From a08ec87b22d670915c2c3ea706546a14121b442c Mon Sep 17 00:00:00 2001 From: QihuAI Design Date: Thu, 23 Jul 2026 05:42:48 +0800 Subject: [PATCH] chore(align): sync Git with running container as authoritative source One-time alignment to close drift between the running container (qihu-aiflow / image qihuaiflow:with-deps) and this Git repository. Container hotfixes previously persisted only via `docker commit` are now captured in Git so Git becomes the single source of truth. Backend: - chat_service.py: accumulated fixes incl. EigenFlux README guidance - chat_context.py / chat_media.py: URL/web-repo context injection - api/url_reader.py: NEW SSRF-safe web/README reader (Issue #725) - skills_admin.py & Skills.tsx: 14-scene capability unification - pgvector_search.py, orchestrator.py, platform_knowledge.py, market.py, admin/models.py: drift sync Frontend: - ChatPage.tsx: collapsible AI think/act block (Issue #724) - MarketPage / EvolutionDashboard / TokenDashboard styling sync Migrations: - 018_normalize_marketapp_scene.sql: fixed capability allowlist (prevents capability wipe on restart) Alignment performed 2026-07-23 05:42:47. No deletions; secrets/venv/node_modules/ caches were not touched (tracked-source-only copy). --- backend/python/app/api/admin/models.py | 8 +- backend/python/app/api/admin/skills_admin.py | 1895 ++-- backend/python/app/api/chat_context.py | 4 +- backend/python/app/api/chat_media.py | 15 +- backend/python/app/api/chat_service.py | 8577 +++++++++-------- backend/python/app/api/chat_utils.py | 774 +- backend/python/app/api/market.py | 10 +- backend/python/app/api/url_reader.py | 163 + backend/python/app/core/orchestrator.py | 790 +- backend/python/app/pgvector_search.py | 750 +- backend/python/app/platform_knowledge.py | 41 +- frontend/src/pages/admin/Skills.tsx | 1358 +-- frontend/src/pages/front/ChatPage.tsx | 96 +- .../src/pages/front/EvolutionDashboard.css | 88 +- .../src/pages/front/EvolutionDashboard.tsx | 28 +- frontend/src/pages/front/MarketPage.css | 20 + frontend/src/pages/front/MarketPage.tsx | 11 +- .../src/pages/front/TokenDashboardPage.css | 8 + migrations/018_normalize_marketapp_scene.sql | 34 +- 19 files changed, 7474 insertions(+), 7196 deletions(-) create mode 100644 backend/python/app/api/url_reader.py diff --git a/backend/python/app/api/admin/models.py b/backend/python/app/api/admin/models.py index 20d4af1..4537c09 100644 --- a/backend/python/app/api/admin/models.py +++ b/backend/python/app/api/admin/models.py @@ -932,7 +932,13 @@ async def test_model_connection( "n": 1, "size": "1024x1024", } - resp = await client.post(image_url, json=payload, headers=headers) + # 与运行时媒体生成一致：用 _proxy_for_url 解析代理并 trust_env=False； + # 图像端点会被 _proxy_for_url 判定为直连(公网 API 直连比代理更稳定) + from ..chat_utils import _proxy_for_url + async with httpx.AsyncClient( + timeout=timeout, proxy=_proxy_for_url(image_url), trust_env=False + ) as img_client: + resp = await img_client.post(image_url, json=payload, headers=headers) latency = int((time.time() - start) * 1000) if resp.status_code == 200: diff --git a/backend/python/app/api/admin/skills_admin.py b/backend/python/app/api/admin/skills_admin.py index 4a06361..296cebc 100644 --- a/backend/python/app/api/admin/skills_admin.py +++ b/backend/python/app/api/admin/skills_admin.py @@ -1,937 +1,958 @@ -""" -技能管理系统 API（管理员） -全平台技能管理：列表/创建/编辑/删除/状态管理/版本历史/发布市场 -""" -import os -import json -import uuid -import logging -from datetime import datetime -from typing import Optional, List - -from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy.orm import Session -from sqlalchemy import desc, func, or_, and_, text - -from ...database import get_db -from ...models import ( - User, EvolutionSkill, SkillVersion, SkillPublishRecord, - EvolutionEvent, TeamSharedSkill, MarketApp, AppInstallation, - AppPurchase, AppReview -) -from ...auth import get_current_admin -from ...middleware.log_service import LogService -from ...config import UPLOAD_DIR - -router = APIRouter() -logger = logging.getLogger(__name__) - -# ============================================================ -# 技能状态定义 -# ============================================================ - -SKILL_STATES = { - "baby": {"label": "萌芽期", "color": "#9CA3AF", "description": "刚创建的技能，需要继续培养"}, - "main": {"label": "主力期", "color": "#10B981", "description": "已成熟可正常使用"}, - "staging": {"label": "灰度期", "color": "#F59E0B", "description": "正在灰度测试中"}, - "frozen": {"label": "已冻结", "color": "#EF4444", "description": "已停止使用"}, -} - -SKILL_CATEGORIES = [ - {"id": "writing", "label": "文字创作"}, - {"id": "coding", "label": "代码能力"}, - {"id": "data_analysis", "label": "数据分析"}, - {"id": "qa", "label": "知识问答"}, - {"id": "automation", "label": "流程自动化"}, - {"id": "creativity", "label": "创意生成"}, - {"id": "communication", "label": "沟通协作"}, - {"id": "reasoning", "label": "逻辑推理"}, - {"id": "other", "label": "其他"}, -] - - -# ============================================================ -# 辅助函数 -# ============================================================ - -def market_app_to_skill_dict(app: "MarketApp") -> dict: - """将市场应用(MarketApp type=skill)转换为与 EvolutionSkill 一致的字典格式""" - status = (getattr(app, "status", None) or "pending").strip() - state = "main" if status == "published" else "baby" - raw_tags = getattr(app, "tags", None) - tags_list = [] - if raw_tags: - try: - tags_list = json.loads(raw_tags) if isinstance(raw_tags, str) else list(raw_tags) - except Exception: - tags_list = [t.strip() for t in str(raw_tags).split(",")] - category = getattr(app, "category", None) or "other" - return { - "id": app.id, - "user_id": getattr(app, "publisher_id", None) or "system", - "skill_name": app.name, - "description": getattr(app, "description", None) or "", - "tags": tags_list, - "category": category, - "category_label": next((c["label"] for c in SKILL_CATEGORIES if c["id"] == category), category or "其他"), - "state": state, - "state_info": SKILL_STATES.get(state, SKILL_STATES["baby"]), - "use_count": getattr(app, "install_count", None) or 0, - "version": 1, - "skill_md_path": None, - "git_commit_sha": None, - "last_used_at": None, - "created_at": app.created_at.isoformat() if getattr(app, "created_at", None) else None, - "updated_at": getattr(app, "updated_at", None).isoformat() if getattr(app, "updated_at", None) else None, - "source": "market", - # 让前端 isPublished(skill) 能正确识别 market 已发布项 - "publish_records": [{"status": "published"}] if status == "published" else [], - } - - -def skill_to_dict(s: EvolutionSkill, include_md: bool = False) -> dict: - """将技能ORM对象转换为字典""" - result = { - "id": s.id, - "user_id": s.user_id, - "skill_name": s.skill_name, - "description": s.description, - "tags": s.tags or [], - "category": s.category, - "category_label": next((c["label"] for c in SKILL_CATEGORIES if c["id"] == s.category), s.category or "其他"), - "state": s.state, - "state_info": SKILL_STATES.get(s.state, SKILL_STATES["baby"]), - "use_count": s.use_count or 0, - "version": s.version or 1, - "skill_md_path": s.skill_md_path, - "git_commit_sha": s.git_commit_sha, - "last_used_at": s.last_used_at.isoformat() if s.last_used_at else None, - "created_at": s.created_at.isoformat() if s.created_at else None, - "updated_at": s.updated_at.isoformat() if s.updated_at else None, - } - - # 读取 SKILL.md 内容 - if include_md and s.skill_md_path and os.path.exists(s.skill_md_path): - try: - with open(s.skill_md_path, 'r', encoding='utf-8') as f: - result["skill_md_content"] = f.read() - except Exception: - result["skill_md_content"] = None - else: - result["skill_md_content"] = None - - return result - - -def version_to_dict(v: SkillVersion) -> dict: - return { - "id": v.id, - "user_id": v.user_id, - "skill_name": v.skill_name, - "version": v.version, - "skill_md_content": v.skill_md_content, - "git_commit_sha": v.git_commit_sha, - "description": v.description, - "created_at": v.created_at.isoformat() if v.created_at else None, - } - - -# ============================================================ -# 管理员技能列表 -# ============================================================ - -@router.get("/") -async def list_skills( - state: str = Query(None, description="技能状态过滤: baby/main/staging/frozen"), - category: str = Query(None, description="技能分类过滤"), - keyword: str = Query(None, description="关键词搜索(名称/描述)"), - user_id: str = Query(None, description="按用户ID筛选"), - sort_by: str = Query("updated_at", description="排序字段"), - order: str = Query("desc", description="排序方向: asc/desc"), - page: int = Query(1, ge=1, description="页码"), - page_size: int = Query(20, ge=1, le=100, description="每页数量"), - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取全平台技能列表（支持分页/过滤/搜索）—— 同时查询 EvolutionSkill + MarketApp(type=skill)""" - logger.info(f"[Admin Skills] {admin_user.username} 列出技能: state={state}, keyword={keyword}") - - result: list = [] - - # ===== 1) 查询 EvolutionSkill ===== - evo_query = db.query(EvolutionSkill) - if state: - evo_query = evo_query.filter(EvolutionSkill.state == state) - if category: - evo_query = evo_query.filter(EvolutionSkill.category == category) - if keyword: - evo_query = evo_query.filter( - or_(EvolutionSkill.skill_name.contains(keyword), EvolutionSkill.description.contains(keyword)) - ) - if user_id: - evo_query = evo_query.filter(EvolutionSkill.user_id == user_id) - - evo_skills = evo_query.all() - user_ids = list(set(s.user_id for s in evo_skills)) - users_map = {u.id: u.username for u in db.query(User.id, User.username).filter(User.id.in_(user_ids)).all()} - for s in evo_skills: - item = skill_to_dict(s) - item["owner_username"] = users_map.get(s.user_id, s.user_id) - item["source"] = "evolution" - result.append(item) - - # ===== 2) 查询 MarketApp (type=skill) ===== - mkt_query = db.query(MarketApp).filter(MarketApp.type == "skill") - if keyword: - mkt_query = mkt_query.filter( - or_(MarketApp.name.contains(keyword), MarketApp.description.contains(keyword)) - ) - if category: - mkt_query = mkt_query.filter(MarketApp.category == category) - # 状态映射：主面板只展示已发布 → state='main'；草稿/移除 → 'baby' - if state == "main": - mkt_query = mkt_query.filter(MarketApp.status == "published") - elif state: - mkt_query = mkt_query.filter(MarketApp.status != "published") # 非发布=草稿等 - - market_apps_rows = mkt_query.all() - for app in market_apps_rows: - item = market_app_to_skill_dict(app) - item["owner_username"] = app.publisher_name or "system" - result.append(item) - - # ===== 3) 合并排序 & 分页 ===== - total = len(result) - reverse = order == "desc" - key_fn = lambda x: x.get(sort_by) or "" - result.sort(key=key_fn, reverse=reverse) - - start = (page - 1) * page_size - end = start + page_size - paged = result[start:end] - - return { - "code": 0, - "data": { - "items": paged, - "total": total, - "page": page, - "page_size": page_size, - "pages": (total + page_size - 1) // page_size if total else 0, - } - } - - -# ============================================================ -# 获取单个技能详情 -# ============================================================ - -@router.get("/{skill_id}") -async def get_skill( - skill_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取技能详情（包含 SKILL.md 内容）""" - skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() - if not skill: - raise HTTPException(status_code=404, detail="技能不存在") - - result = skill_to_dict(skill, include_md=True) - - # 获取所属用户信息 - user = db.query(User).filter(User.id == skill.user_id).first() - if user: - result["owner_username"] = user.username - result["owner_email"] = user.email - - # 获取发布记录 - pub_records = db.query(SkillPublishRecord).filter( - SkillPublishRecord.skill_id == skill_id - ).order_by(desc(SkillPublishRecord.created_at)).limit(10).all() - result["publish_records"] = [{ - "id": p.id, - "price": float(p.price) if p.price else 0, - "market_description": p.market_description, - "status": p.status, - "published_at": p.published_at.isoformat() if p.published_at else None, - "created_at": p.created_at.isoformat() if p.created_at else None, - } for p in pub_records] - - # 获取最新版本号 - latest_version = db.query(func.max(SkillVersion.version)).filter( - SkillVersion.skill_id == skill_id - ).scalar() - result["latest_version"] = latest_version or skill.version - - return {"code": 0, "data": result} - - -# ============================================================ -# 创建技能 -# ============================================================ - -class CreateSkillRequest: - def __init__(self, skill_name: str = None, description: str = None, - tags: List[str] = None, category: str = None, - skill_md_content: str = None, user_id: str = None): - self.skill_name = skill_name - self.description = description - self.tags = tags or [] - self.category = category - self.skill_md_content = skill_md_content - self.user_id = user_id - - -from pydantic import BaseModel, Field - - -class CreateSkillSchema(BaseModel): - skill_name: str = Field(..., min_length=1, max_length=200, description="技能名称") - user_id: str = Field(..., description="所属用户ID") - description: Optional[str] = Field(None, description="技能描述") - tags: Optional[List[str]] = Field(default_factory=list, description="标签列表") - category: Optional[str] = Field(None, description="技能分类") - skill_md_content: Optional[str] = Field(None, description="SKILL.md 内容") - - -@router.post("/") -async def create_skill( - req: CreateSkillSchema, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """管理员创建技能""" - logger.info(f"[Admin Skills] {admin_user.username} 创建技能: {req.skill_name}") - - # 检查技能名是否已存在 - existing = db.query(EvolutionSkill).filter( - EvolutionSkill.user_id == req.user_id, - EvolutionSkill.skill_name == req.skill_name - ).first() - if existing: - raise HTTPException(status_code=400, detail="该用户已存在同名技能") - - # 验证用户存在 - user = db.query(User).filter(User.id == req.user_id).first() - if not user: - raise HTTPException(status_code=400, detail="用户不存在") - - # 生成技能ID - skill_id = str(uuid.uuid4()) - - # 保存 SKILL.md 文件 - skill_md_path = None - if req.skill_md_content: - skill_dir = os.path.join(UPLOAD_DIR, "skills", req.user_id) - os.makedirs(skill_dir, exist_ok=True) - safe_name = req.skill_name.replace("/", "_").replace("\\", "_") - skill_md_path = os.path.join(skill_dir, f"{safe_name}.md") - try: - with open(skill_md_path, 'w', encoding='utf-8') as f: - f.write(req.skill_md_content) - except Exception as e: - logger.warning(f"保存 SKILL.md 失败: {e}") - - # 创建技能记录 - skill = EvolutionSkill( - id=skill_id, - user_id=req.user_id, - skill_name=req.skill_name, - description=req.description, - tags=req.tags or [], - category=req.category, - state="baby", - use_count=0, - version=1, - skill_md_path=skill_md_path, - git_commit_sha=None, - ) - db.add(skill) - - # 记录初始版本 - if req.skill_md_content: - sv = SkillVersion( - id=str(uuid.uuid4()), - user_id=req.user_id, - skill_name=req.skill_name, - version=1, - skill_md_content=req.skill_md_content, - git_commit_sha=None, - description="初始版本", - ) - db.add(sv) - - # 记录事件 - ev = EvolutionEvent( - id=str(uuid.uuid4()), - user_id=req.user_id, - skill_name=req.skill_name, - event_type="created", - description=f"管理员 {admin_user.username} 创建技能 {req.skill_name}", - version_before=None, - version_after=1, - ) - db.add(ev) - - db.commit() - db.refresh(skill) - - LogService.log_operation( - user_id=admin_user.id, - username=admin_user.username, - action="create_skill", - target="evolution_skills", - detail=f"创建技能 {req.skill_name}（归属用户: {user.username}）", - ) - - return {"code": 0, "data": skill_to_dict(skill, include_md=True)} - - -# ============================================================ -# 更新技能 -# ============================================================ - -class UpdateSkillSchema(BaseModel): - skill_name: Optional[str] = Field(None, min_length=1, max_length=200) - description: Optional[str] = None - tags: Optional[List[str]] = None - category: Optional[str] = None - skill_md_content: Optional[str] = None - - -@router.put("/{skill_id}") -async def update_skill( - skill_id: str, - req: UpdateSkillSchema, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """更新技能信息""" - skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() - if not skill: - raise HTTPException(status_code=404, detail="技能不存在") - - old_version = skill.version - update_data = req.model_dump(exclude_none=True) - - # 处理 SKILL.md 内容 - if "skill_md_content" in update_data: - content = update_data.pop("skill_md_content") - if content is not None: - skill_dir = os.path.join(UPLOAD_DIR, "skills", skill.user_id) - os.makedirs(skill_dir, exist_ok=True) - safe_name = skill.skill_name.replace("/", "_").replace("\\", "_") - skill_md_path = os.path.join(skill_dir, f"{safe_name}.md") - try: - with open(skill_md_path, 'w', encoding='utf-8') as f: - f.write(content) - skill.skill_md_path = skill_md_path - except Exception as e: - logger.warning(f"保存 SKILL.md 失败: {e}") - - # 记录新版本 - new_version = old_version + 1 - sv = SkillVersion( - id=str(uuid.uuid4()), - user_id=skill.user_id, - skill_name=skill.skill_name, - version=new_version, - skill_md_content=content, - git_commit_sha=None, - description="管理员更新", - ) - db.add(sv) - skill.version = new_version - - # 记录事件 - ev = EvolutionEvent( - id=str(uuid.uuid4()), - user_id=skill.user_id, - skill_name=skill.skill_name, - event_type="edited", - description=f"管理员 {admin_user.username} 更新技能 {skill.skill_name}", - version_before=old_version, - version_after=new_version, - ) - db.add(ev) - - # 更新其他字段 - for key, value in update_data.items(): - if key not in ("skill_md_content",) and hasattr(skill, key): - setattr(skill, key, value) - - skill.updated_at = datetime.now() - db.commit() - db.refresh(skill) - - LogService.log_operation( - user_id=admin_user.id, - username=admin_user.username, - action="update_skill", - target="evolution_skills", - detail=f"更新技能 {skill.skill_name}（版本 {old_version} → {skill.version}）", - ) - - return {"code": 0, "data": skill_to_dict(skill, include_md=True)} - - -# ============================================================ -# 批量删除技能（用于一次性清空技能市场） -# ============================================================ - - -class BatchDeleteSkillsRequest(BaseModel): - ids: list[str] = [] # 要删除的技能 ID 列表；传空列表则删除所有 - - -@router.post("/batch-delete") -async def batch_delete_skills( - req: BatchDeleteSkillsRequest, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """批量删除技能。ids 为空则删除全部技能。""" - if req.ids: - skills = db.query(EvolutionSkill).filter(EvolutionSkill.id.in_(req.ids)).all() - else: - skills = db.query(EvolutionSkill).all() - - deleted_count = 0 - for skill in skills: - skill_name = skill.skill_name - # 删除版本历史 - db.query(SkillVersion).filter(SkillVersion.skill_name == skill_name).delete() - # 删除发布记录 - db.query(SkillPublishRecord).filter(SkillPublishRecord.skill_id == skill.id).delete() - # 删除团队共享 - db.query(TeamSharedSkill).filter(TeamSharedSkill.skill_id == skill.id).delete() - # 同步清理市场应用表中的技能 - market_app = db.query(MarketApp).filter( - MarketApp.type == "skill", - MarketApp.name == skill_name, - ).first() - if market_app: - # 清理关联的安装和购买记录 - db.query(AppInstallation).filter(AppInstallation.app_id == market_app.id).delete() - db.query(AppPurchase).filter(AppPurchase.app_id == market_app.id).delete() - db.query(AppReview).filter(AppReview.app_id == market_app.id).delete() - db.delete(market_app) - db.delete(skill) - deleted_count += 1 - - db.commit() - return {"code": 0, "data": {"deleted": deleted_count}, "message": f"已删除 {deleted_count} 个技能"} - - -# ============================================================ -# 删除技能（单个） -# ============================================================ - -@router.delete("/{skill_id}") -async def delete_skill( - skill_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """删除技能（支持 EvolutionSkill 和 MarketApp 两种来源）""" - # 1) 先尝试 EvolutionSkill - skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() - if skill: - skill_name = skill.skill_name - user_id = skill.user_id - - # 记录事件 - ev = EvolutionEvent( - id=str(uuid.uuid4()), - user_id=user_id, - skill_name=skill_name, - event_type="deleted", - description=f"管理员 {admin_user.username} 删除技能 {skill_name}", - ) - db.add(ev) - - # 删除版本历史 - db.query(SkillVersion).filter(SkillVersion.skill_name == skill_name).delete() - - # 删除发布记录 - db.query(SkillPublishRecord).filter(SkillPublishRecord.skill_id == skill_id).delete() - - # 删除团队共享记录 - db.query(TeamSharedSkill).filter(TeamSharedSkill.skill_id == skill_id).delete() - - # 同步清理市场应用表中的同名技能 - market_app = db.query(MarketApp).filter( - MarketApp.type == "skill", - MarketApp.name == skill_name, - ).first() - if market_app: - db.query(AppInstallation).filter(AppInstallation.app_id == market_app.id).delete() - db.query(AppPurchase).filter(AppPurchase.app_id == market_app.id).delete() - db.query(AppReview).filter(AppReview.app_id == market_app.id).delete() - db.delete(market_app) - - db.delete(skill) - db.commit() - - LogService.log_operation( - user_id=admin_user.id, - username=admin_user.username, - action="delete_skill", - target="evolution_skills", - detail=f"删除技能 {skill_name}", - ) - - return {"code": 0, "message": f"技能 {skill_name} 已删除"} - - # 2) 再尝试 MarketApp 中导入的技能（type=skill） - market_app = db.query(MarketApp).filter( - MarketApp.type == "skill", - MarketApp.id == skill_id, - ).first() - if market_app: - skill_name = market_app.name - # 清理关联记录 - db.query(AppInstallation).filter(AppInstallation.app_id == market_app.id).delete() - db.query(AppPurchase).filter(AppPurchase.app_id == market_app.id).delete() - db.query(AppReview).filter(AppReview.app_id == market_app.id).delete() - db.delete(market_app) - db.commit() - - LogService.log_operation( - user_id=admin_user.id, - username=admin_user.username, - action="delete_skill", - target="market_apps", - detail=f"删除市场技能 {skill_name}", - ) - - return {"code": 0, "message": f"技能 {skill_name} 已删除"} - - # 3) 都未找到 - raise HTTPException(status_code=404, detail="技能不存在") - - -# ============================================================ -# 修改技能状态 -# ============================================================ - -class ChangeStateSchema(BaseModel): - state: str = Field(..., description="目标状态: baby/main/staging/frozen") - - -@router.put("/{skill_id}/state") -async def change_skill_state( - skill_id: str, - req: ChangeStateSchema, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """修改技能状态""" - if req.state not in SKILL_STATES: - raise HTTPException(status_code=400, detail=f"无效状态，可选值: {list(SKILL_STATES.keys())}") - - skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() - if not skill: - raise HTTPException(status_code=404, detail="技能不存在") - - old_state = skill.state - skill.state = req.state - skill.updated_at = datetime.now() - - # 记录状态变更事件 - ev_type = "promoted" if req.state in ("main", "staging") else "frozen" if req.state == "frozen" else "unfrozen" - ev = EvolutionEvent( - id=str(uuid.uuid4()), - user_id=skill.user_id, - skill_name=skill.skill_name, - event_type=ev_type, - description=f"管理员 {admin_user.username} 将技能状态从 {old_state} 变更为 {req.state}", - ) - db.add(ev) - db.commit() - db.refresh(skill) - - LogService.log_operation( - user_id=admin_user.id, - username=admin_user.username, - action="change_skill_state", - target="evolution_skills", - detail=f"修改技能 {skill.skill_name} 状态: {old_state} → {req.state}", - ) - - return {"code": 0, "data": skill_to_dict(skill)} - - -# ============================================================ -# 获取版本历史 -# ============================================================ - -@router.get("/{skill_id}/versions") -async def get_skill_versions( - skill_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取技能版本历史""" - skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() - if not skill: - raise HTTPException(status_code=404, detail="技能不存在") - - versions = db.query(SkillVersion).filter( - SkillVersion.skill_name == skill.skill_name, - SkillVersion.user_id == skill.user_id, - ).order_by(desc(SkillVersion.version)).all() - - return { - "code": 0, - "data": { - "skill_id": skill.id, - "skill_name": skill.skill_name, - "current_version": skill.version, - "versions": [version_to_dict(v) for v in versions], - } - } - - -# ============================================================ -# 回滚到指定版本 -# ============================================================ - -class RollbackSchema(BaseModel): - version: int = Field(..., ge=1, description="目标版本号") - - -@router.post("/{skill_id}/rollback") -async def rollback_skill_version( - skill_id: str, - req: RollbackSchema, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """回滚技能到指定版本""" - skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() - if not skill: - raise HTTPException(status_code=404, detail="技能不存在") - - target_version = db.query(SkillVersion).filter( - SkillVersion.skill_name == skill.skill_name, - SkillVersion.user_id == skill.user_id, - SkillVersion.version == req.version, - ).first() - - if not target_version: - raise HTTPException(status_code=404, detail=f"版本 {req.version} 不存在") - - old_version = skill.version - - # 更新 SKILL.md 文件 - if target_version.skill_md_content: - if skill.skill_md_path: - try: - with open(skill.skill_md_path, 'w', encoding='utf-8') as f: - f.write(target_version.skill_md_content) - except Exception as e: - logger.warning(f"回滚保存 SKILL.md 失败: {e}") - - # 记录新版本（基于目标版本） - new_version = old_version + 1 - sv = SkillVersion( - id=str(uuid.uuid4()), - user_id=skill.user_id, - skill_name=skill.skill_name, - version=new_version, - skill_md_content=target_version.skill_md_content, - git_commit_sha=None, - description=f"回滚到版本 {req.version}（管理员操作）", - ) - db.add(sv) - - skill.version = new_version - skill.updated_at = datetime.now() - - ev = EvolutionEvent( - id=str(uuid.uuid4()), - user_id=skill.user_id, - skill_name=skill.skill_name, - event_type="edited", - description=f"管理员 {admin_user.username} 回滚技能 {skill.skill_name} 从版本 {old_version} 到 {req.version}（新版本 {new_version}）", - version_before=old_version, - version_after=new_version, - ) - db.add(ev) - - db.commit() - db.refresh(skill) - - LogService.log_operation( - user_id=admin_user.id, - username=admin_user.username, - action="rollback_skill", - target="evolution_skills", - detail=f"回滚技能 {skill.skill_name} 到版本 {req.version}（新版本 {new_version}）", - ) - - return {"code": 0, "data": skill_to_dict(skill, include_md=True)} - - -# ============================================================ -# 发布到应用市场 -# ============================================================ - -class PublishSkillSchema(BaseModel): - price: float = Field(0.0, ge=0, description="定价（0表示免费）") - market_description: str = Field("", description="市场描述") - - -@router.post("/{skill_id}/publish") -async def publish_skill( - skill_id: str, - req: PublishSkillSchema, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """发布技能到应用市场""" - skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() - if not skill: - raise HTTPException(status_code=404, detail="技能不存在") - - # 检查是否已发布 - existing = db.query(SkillPublishRecord).filter( - SkillPublishRecord.skill_id == skill_id, - SkillPublishRecord.status == "published" - ).first() - - if existing: - # 更新现有发布 - existing.price = req.price - existing.market_description = req.market_description - msg = "更新发布信息" - else: - # 创建发布记录 - record = SkillPublishRecord( - id=str(uuid.uuid4()), - skill_id=skill_id, - skill_name=skill.skill_name, - user_id=skill.user_id, - price=req.price, - market_description=req.market_description, - status="published", - ) - db.add(record) - msg = "发布成功" - - db.commit() - - LogService.log_operation( - user_id=admin_user.id, - username=admin_user.username, - action="publish_skill", - target="skill_publish_records", - detail=f"{msg}: {skill.skill_name}，定价: ¥{req.price}", - ) - - return {"code": 0, "message": msg, "data": {"skill_id": skill_id, "price": req.price}} - - -# ============================================================ -# 下架技能 -# ============================================================ - -@router.post("/{skill_id}/unpublish") -async def unpublish_skill( - skill_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """从应用市场下架技能""" - skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() - if not skill: - raise HTTPException(status_code=404, detail="技能不存在") - - updated = db.query(SkillPublishRecord).filter( - SkillPublishRecord.skill_id == skill_id, - SkillPublishRecord.status == "published" - ).update({"status": "unpublished"}) - db.commit() - - if updated == 0: - raise HTTPException(status_code=400, detail="该技能未发布到市场") - - LogService.log_operation( - user_id=admin_user.id, - username=admin_user.username, - action="unpublish_skill", - target="skill_publish_records", - detail=f"下架技能: {skill.skill_name}", - ) - - return {"code": 0, "message": f"技能 {skill.skill_name} 已下架"} - - -# ============================================================ -# 获取统计数据 -# ============================================================ - -@router.get("/stats/overview") -async def get_skills_stats( - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取技能统计概览（EvolutionSkill + MarketApp type=skill 合并统计）""" - # EvolutionSkill 维度 - evo_total = db.query(EvolutionSkill).count() - evo_by_state = dict( - db.query(EvolutionSkill.state, func.count(EvolutionSkill.id)) - .group_by(EvolutionSkill.state).all() - ) - evo_by_category = dict( - db.query(EvolutionSkill.category, func.count(EvolutionSkill.id)) - .filter(EvolutionSkill.category.isnot(None)) - .group_by(EvolutionSkill.category).all() - ) - - # MarketApp(type=skill) 维度：状态映射 published→main, 其它→baby - market_total = db.query(MarketApp).filter(MarketApp.type == "skill").count() - market_rows = db.query(MarketApp.status, func.count(MarketApp.id)).filter( - MarketApp.type == "skill" - ).group_by(MarketApp.status).all() - market_published = sum(cnt for status, cnt in market_rows if (status or "") == "published") - # 各 category 计数（market 表 category 在 SQL 列有，ORM 无此属性，用 SQL 拿） - market_by_category = {} - try: - market_by_category_rows = db.execute( - text("SELECT category, COUNT(*) FROM market_apps WHERE type='skill' AND category IS NOT NULL GROUP BY category") - ).fetchall() - market_by_category = {cat or "other": cnt for cat, cnt in market_by_category_rows} - except Exception: - pass - - # 合并 - total_skills = evo_total + market_total - by_state = {k: evo_by_state.get(k, 0) for k in SKILL_STATES} - by_state["main"] = by_state.get("main", 0) + market_published - by_state["baby"] = by_state.get("baby", 0) + (market_total - market_published) - by_category = {} - for c in (evo_by_category or {}).items(): - by_category[c[0]] = by_category.get(c[0], 0) + c[1] - for c in (market_by_category or {}).items(): - by_category[c[0]] = by_category.get(c[0], 0) + c[1] - - # 原始发布记录 + market 已发布 = 全部已发布 - skill_record_published = db.query(SkillPublishRecord).filter( - SkillPublishRecord.status == "published" - ).count() - total_published = skill_record_published + market_published - total_versions = db.query(SkillVersion).count() - - return { - "code": 0, - "data": { - "total_skills": total_skills, - "by_state": by_state, - "by_category": by_category, - "total_published": total_published, - "total_versions": total_versions, - } - } +""" +技能管理系统 API（管理员） +全平台技能管理：列表/创建/编辑/删除/状态管理/版本历史/发布市场 +""" +import os +import json +import uuid +import logging +from datetime import datetime +from typing import Optional, List + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy.orm import Session +from sqlalchemy import desc, func, or_, and_, text + +from ...database import get_db +from ...models import ( + User, EvolutionSkill, SkillVersion, SkillPublishRecord, + EvolutionEvent, TeamSharedSkill, MarketApp, AppInstallation, + AppPurchase, AppReview +) +from ...auth import get_current_admin +from ...middleware.log_service import LogService +from ...config import UPLOAD_DIR + +router = APIRouter() +logger = logging.getLogger(__name__) + +# ============================================================ +# 技能状态定义 +# ============================================================ + +SKILL_STATES = { + "baby": {"label": "萌芽期", "color": "#9CA3AF", "description": "刚创建的技能，需要继续培养"}, + "main": {"label": "主力期", "color": "#10B981", "description": "已成熟可正常使用"}, + "staging": {"label": "灰度期", "color": "#F59E0B", "description": "正在灰度测试中"}, + "frozen": {"label": "已冻结", "color": "#EF4444", "description": "已停止使用"}, +} + +# 业务场景分类（与前端「应用市场」保持一致，14 类岗位/职能视角） +# 统一标准：以前台 MARKET_SCENES 为准，后台技能管理也按此 14 类展示与筛选。 +SKILL_CATEGORIES = [ + {"id": "design", "label": "🎨 做设计"}, + {"id": "frontend", "label": "💻 写前端"}, + {"id": "mobile", "label": "📱 做移动端"}, + {"id": "backend", "label": "🧪 后端/全栈"}, + {"id": "qa", "label": "🧪 测试质量"}, + {"id": "search", "label": "🔍 搜索信息"}, + {"id": "docs", "label": "✍️ 写文档"}, + {"id": "marketing", "label": "📢 推广获客"}, + {"id": "workflow", "label": "🔄 工作流"}, + {"id": "secops", "label": "🛡️ 安全运维"}, + {"id": "learning", "label": "🎓 学习教学"}, + {"id": "video", "label": "🎬 视频多媒体"}, + {"id": "collab", "label": "🏢 办公协作"}, + {"id": "ai", "label": "🛠️ AI能力"}, +] +_SCENE_IDS = {c["id"] for c in SKILL_CATEGORIES} +# 旧 9 类 taxonomy -> 14 场景 的兜底映射（仅用于历史数据 capability 缺失时） +_CATEGORY_TO_SCENE = { + "writing": "docs", "coding": "backend", "data_analysis": "backend", + "qa": "qa", "automation": "workflow", "creativity": "design", + "communication": "marketing", "reasoning": "search", "other": "workflow", +} + + +# ============================================================ +# 辅助函数 +# ============================================================ + +def market_app_to_skill_dict(app: "MarketApp") -> dict: + """将市场应用(MarketApp type=skill)转换为与 EvolutionSkill 一致的字典格式""" + status = (getattr(app, "status", None) or "pending").strip() + state = "main" if status == "published" else "baby" + raw_tags = getattr(app, "tags", None) + tags_list = [] + if raw_tags: + try: + tags_list = json.loads(raw_tags) if isinstance(raw_tags, str) else list(raw_tags) + except Exception: + tags_list = [t.strip() for t in str(raw_tags).split(",")] + # 以业务场景(capability, 14 类)为分类标准，与前端应用市场统一； + # capability 缺失时回退到旧 9 类 category 映射出的场景。 + raw_scene = getattr(app, "capability", None) + if raw_scene in _SCENE_IDS: + category = raw_scene + else: + category = _CATEGORY_TO_SCENE.get(getattr(app, "category", None), "workflow") + return { + "id": app.id, + "user_id": getattr(app, "publisher_id", None) or "system", + "skill_name": app.name, + "description": getattr(app, "description", None) or "", + "tags": tags_list, + "category": category, + "category_label": next((c["label"] for c in SKILL_CATEGORIES if c["id"] == category), category or "其他"), + "state": state, + "state_info": SKILL_STATES.get(state, SKILL_STATES["baby"]), + "use_count": getattr(app, "install_count", None) or 0, + "version": 1, + "skill_md_path": None, + "git_commit_sha": None, + "last_used_at": None, + "created_at": app.created_at.isoformat() if getattr(app, "created_at", None) else None, + "updated_at": getattr(app, "updated_at", None).isoformat() if getattr(app, "updated_at", None) else None, + "source": "market", + # 让前端 isPublished(skill) 能正确识别 market 已发布项 + "publish_records": [{"status": "published"}] if status == "published" else [], + } + + +def skill_to_dict(s: EvolutionSkill, include_md: bool = False) -> dict: + """将技能ORM对象转换为字典""" + result = { + "id": s.id, + "user_id": s.user_id, + "skill_name": s.skill_name, + "description": s.description, + "tags": s.tags or [], + "category": s.category, + "category_label": next((c["label"] for c in SKILL_CATEGORIES if c["id"] == s.category), s.category or "其他"), + "state": s.state, + "state_info": SKILL_STATES.get(s.state, SKILL_STATES["baby"]), + "use_count": s.use_count or 0, + "version": s.version or 1, + "skill_md_path": s.skill_md_path, + "git_commit_sha": s.git_commit_sha, + "last_used_at": s.last_used_at.isoformat() if s.last_used_at else None, + "created_at": s.created_at.isoformat() if s.created_at else None, + "updated_at": s.updated_at.isoformat() if s.updated_at else None, + } + + # 读取 SKILL.md 内容 + if include_md and s.skill_md_path and os.path.exists(s.skill_md_path): + try: + with open(s.skill_md_path, 'r', encoding='utf-8') as f: + result["skill_md_content"] = f.read() + except Exception: + result["skill_md_content"] = None + else: + result["skill_md_content"] = None + + return result + + +def version_to_dict(v: SkillVersion) -> dict: + return { + "id": v.id, + "user_id": v.user_id, + "skill_name": v.skill_name, + "version": v.version, + "skill_md_content": v.skill_md_content, + "git_commit_sha": v.git_commit_sha, + "description": v.description, + "created_at": v.created_at.isoformat() if v.created_at else None, + } + + +# ============================================================ +# 管理员技能列表 +# ============================================================ + +@router.get("/") +async def list_skills( + state: str = Query(None, description="技能状态过滤: baby/main/staging/frozen"), + category: str = Query(None, description="技能分类过滤"), + keyword: str = Query(None, description="关键词搜索(名称/描述)"), + user_id: str = Query(None, description="按用户ID筛选"), + sort_by: str = Query("updated_at", description="排序字段"), + order: str = Query("desc", description="排序方向: asc/desc"), + page: int = Query(1, ge=1, description="页码"), + page_size: int = Query(20, ge=1, le=100, description="每页数量"), + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取全平台技能列表（支持分页/过滤/搜索）—— 同时查询 EvolutionSkill + MarketApp(type=skill)""" + logger.info(f"[Admin Skills] {admin_user.username} 列出技能: state={state}, keyword={keyword}") + + result: list = [] + + # ===== 1) 查询 EvolutionSkill ===== + evo_query = db.query(EvolutionSkill) + if state: + evo_query = evo_query.filter(EvolutionSkill.state == state) + if category: + evo_query = evo_query.filter(EvolutionSkill.category == category) + if keyword: + evo_query = evo_query.filter( + or_(EvolutionSkill.skill_name.contains(keyword), EvolutionSkill.description.contains(keyword)) + ) + if user_id: + evo_query = evo_query.filter(EvolutionSkill.user_id == user_id) + + evo_skills = evo_query.all() + user_ids = list(set(s.user_id for s in evo_skills)) + users_map = {u.id: u.username for u in db.query(User.id, User.username).filter(User.id.in_(user_ids)).all()} + for s in evo_skills: + item = skill_to_dict(s) + item["owner_username"] = users_map.get(s.user_id, s.user_id) + item["source"] = "evolution" + result.append(item) + + # ===== 2) 查询 MarketApp (type=skill) ===== + mkt_query = db.query(MarketApp).filter(MarketApp.type == "skill") + if keyword: + mkt_query = mkt_query.filter( + or_(MarketApp.name.contains(keyword), MarketApp.description.contains(keyword)) + ) + if category: + # 分类筛选统一按业务场景(capability, 14 类)进行，与前端应用市场一致 + mkt_query = mkt_query.filter(MarketApp.capability == category) + # 状态映射：主面板只展示已发布 → state='main'；草稿/移除 → 'baby' + if state == "main": + mkt_query = mkt_query.filter(MarketApp.status == "published") + elif state: + mkt_query = mkt_query.filter(MarketApp.status != "published") # 非发布=草稿等 + + market_apps_rows = mkt_query.all() + for app in market_apps_rows: + item = market_app_to_skill_dict(app) + item["owner_username"] = app.publisher_name or "system" + result.append(item) + + # ===== 3) 合并排序 & 分页 ===== + total = len(result) + reverse = order == "desc" + key_fn = lambda x: x.get(sort_by) or "" + result.sort(key=key_fn, reverse=reverse) + + start = (page - 1) * page_size + end = start + page_size + paged = result[start:end] + + return { + "code": 0, + "data": { + "items": paged, + "total": total, + "page": page, + "page_size": page_size, + "pages": (total + page_size - 1) // page_size if total else 0, + } + } + + +# ============================================================ +# 获取单个技能详情 +# ============================================================ + +@router.get("/{skill_id}") +async def get_skill( + skill_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取技能详情（包含 SKILL.md 内容）""" + skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() + if not skill: + raise HTTPException(status_code=404, detail="技能不存在") + + result = skill_to_dict(skill, include_md=True) + + # 获取所属用户信息 + user = db.query(User).filter(User.id == skill.user_id).first() + if user: + result["owner_username"] = user.username + result["owner_email"] = user.email + + # 获取发布记录 + pub_records = db.query(SkillPublishRecord).filter( + SkillPublishRecord.skill_id == skill_id + ).order_by(desc(SkillPublishRecord.created_at)).limit(10).all() + result["publish_records"] = [{ + "id": p.id, + "price": float(p.price) if p.price else 0, + "market_description": p.market_description, + "status": p.status, + "published_at": p.published_at.isoformat() if p.published_at else None, + "created_at": p.created_at.isoformat() if p.created_at else None, + } for p in pub_records] + + # 获取最新版本号 + latest_version = db.query(func.max(SkillVersion.version)).filter( + SkillVersion.skill_id == skill_id + ).scalar() + result["latest_version"] = latest_version or skill.version + + return {"code": 0, "data": result} + + +# ============================================================ +# 创建技能 +# ============================================================ + +class CreateSkillRequest: + def __init__(self, skill_name: str = None, description: str = None, + tags: List[str] = None, category: str = None, + skill_md_content: str = None, user_id: str = None): + self.skill_name = skill_name + self.description = description + self.tags = tags or [] + self.category = category + self.skill_md_content = skill_md_content + self.user_id = user_id + + +from pydantic import BaseModel, Field + + +class CreateSkillSchema(BaseModel): + skill_name: str = Field(..., min_length=1, max_length=200, description="技能名称") + user_id: str = Field(..., description="所属用户ID") + description: Optional[str] = Field(None, description="技能描述") + tags: Optional[List[str]] = Field(default_factory=list, description="标签列表") + category: Optional[str] = Field(None, description="技能分类") + skill_md_content: Optional[str] = Field(None, description="SKILL.md 内容") + + +@router.post("/") +async def create_skill( + req: CreateSkillSchema, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """管理员创建技能""" + logger.info(f"[Admin Skills] {admin_user.username} 创建技能: {req.skill_name}") + + # 检查技能名是否已存在 + existing = db.query(EvolutionSkill).filter( + EvolutionSkill.user_id == req.user_id, + EvolutionSkill.skill_name == req.skill_name + ).first() + if existing: + raise HTTPException(status_code=400, detail="该用户已存在同名技能") + + # 验证用户存在 + user = db.query(User).filter(User.id == req.user_id).first() + if not user: + raise HTTPException(status_code=400, detail="用户不存在") + + # 生成技能ID + skill_id = str(uuid.uuid4()) + + # 保存 SKILL.md 文件 + skill_md_path = None + if req.skill_md_content: + skill_dir = os.path.join(UPLOAD_DIR, "skills", req.user_id) + os.makedirs(skill_dir, exist_ok=True) + safe_name = req.skill_name.replace("/", "_").replace("\\", "_") + skill_md_path = os.path.join(skill_dir, f"{safe_name}.md") + try: + with open(skill_md_path, 'w', encoding='utf-8') as f: + f.write(req.skill_md_content) + except Exception as e: + logger.warning(f"保存 SKILL.md 失败: {e}") + + # 创建技能记录 + skill = EvolutionSkill( + id=skill_id, + user_id=req.user_id, + skill_name=req.skill_name, + description=req.description, + tags=req.tags or [], + category=req.category, + state="baby", + use_count=0, + version=1, + skill_md_path=skill_md_path, + git_commit_sha=None, + ) + db.add(skill) + + # 记录初始版本 + if req.skill_md_content: + sv = SkillVersion( + id=str(uuid.uuid4()), + user_id=req.user_id, + skill_name=req.skill_name, + version=1, + skill_md_content=req.skill_md_content, + git_commit_sha=None, + description="初始版本", + ) + db.add(sv) + + # 记录事件 + ev = EvolutionEvent( + id=str(uuid.uuid4()), + user_id=req.user_id, + skill_name=req.skill_name, + event_type="created", + description=f"管理员 {admin_user.username} 创建技能 {req.skill_name}", + version_before=None, + version_after=1, + ) + db.add(ev) + + db.commit() + db.refresh(skill) + + LogService.log_operation( + user_id=admin_user.id, + username=admin_user.username, + action="create_skill", + target="evolution_skills", + detail=f"创建技能 {req.skill_name}（归属用户: {user.username}）", + ) + + return {"code": 0, "data": skill_to_dict(skill, include_md=True)} + + +# ============================================================ +# 更新技能 +# ============================================================ + +class UpdateSkillSchema(BaseModel): + skill_name: Optional[str] = Field(None, min_length=1, max_length=200) + description: Optional[str] = None + tags: Optional[List[str]] = None + category: Optional[str] = None + skill_md_content: Optional[str] = None + + +@router.put("/{skill_id}") +async def update_skill( + skill_id: str, + req: UpdateSkillSchema, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """更新技能信息""" + skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() + if not skill: + raise HTTPException(status_code=404, detail="技能不存在") + + old_version = skill.version + update_data = req.model_dump(exclude_none=True) + + # 处理 SKILL.md 内容 + if "skill_md_content" in update_data: + content = update_data.pop("skill_md_content") + if content is not None: + skill_dir = os.path.join(UPLOAD_DIR, "skills", skill.user_id) + os.makedirs(skill_dir, exist_ok=True) + safe_name = skill.skill_name.replace("/", "_").replace("\\", "_") + skill_md_path = os.path.join(skill_dir, f"{safe_name}.md") + try: + with open(skill_md_path, 'w', encoding='utf-8') as f: + f.write(content) + skill.skill_md_path = skill_md_path + except Exception as e: + logger.warning(f"保存 SKILL.md 失败: {e}") + + # 记录新版本 + new_version = old_version + 1 + sv = SkillVersion( + id=str(uuid.uuid4()), + user_id=skill.user_id, + skill_name=skill.skill_name, + version=new_version, + skill_md_content=content, + git_commit_sha=None, + description="管理员更新", + ) + db.add(sv) + skill.version = new_version + + # 记录事件 + ev = EvolutionEvent( + id=str(uuid.uuid4()), + user_id=skill.user_id, + skill_name=skill.skill_name, + event_type="edited", + description=f"管理员 {admin_user.username} 更新技能 {skill.skill_name}", + version_before=old_version, + version_after=new_version, + ) + db.add(ev) + + # 更新其他字段 + for key, value in update_data.items(): + if key not in ("skill_md_content",) and hasattr(skill, key): + setattr(skill, key, value) + + skill.updated_at = datetime.now() + db.commit() + db.refresh(skill) + + LogService.log_operation( + user_id=admin_user.id, + username=admin_user.username, + action="update_skill", + target="evolution_skills", + detail=f"更新技能 {skill.skill_name}（版本 {old_version} → {skill.version}）", + ) + + return {"code": 0, "data": skill_to_dict(skill, include_md=True)} + + +# ============================================================ +# 批量删除技能（用于一次性清空技能市场） +# ============================================================ + + +class BatchDeleteSkillsRequest(BaseModel): + ids: list[str] = [] # 要删除的技能 ID 列表；传空列表则删除所有 + + +@router.post("/batch-delete") +async def batch_delete_skills( + req: BatchDeleteSkillsRequest, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """批量删除技能。ids 为空则删除全部技能。""" + if req.ids: + skills = db.query(EvolutionSkill).filter(EvolutionSkill.id.in_(req.ids)).all() + else: + skills = db.query(EvolutionSkill).all() + + deleted_count = 0 + for skill in skills: + skill_name = skill.skill_name + # 删除版本历史 + db.query(SkillVersion).filter(SkillVersion.skill_name == skill_name).delete() + # 删除发布记录 + db.query(SkillPublishRecord).filter(SkillPublishRecord.skill_id == skill.id).delete() + # 删除团队共享 + db.query(TeamSharedSkill).filter(TeamSharedSkill.skill_id == skill.id).delete() + # 同步清理市场应用表中的技能 + market_app = db.query(MarketApp).filter( + MarketApp.type == "skill", + MarketApp.name == skill_name, + ).first() + if market_app: + # 清理关联的安装和购买记录 + db.query(AppInstallation).filter(AppInstallation.app_id == market_app.id).delete() + db.query(AppPurchase).filter(AppPurchase.app_id == market_app.id).delete() + db.query(AppReview).filter(AppReview.app_id == market_app.id).delete() + db.delete(market_app) + db.delete(skill) + deleted_count += 1 + + db.commit() + return {"code": 0, "data": {"deleted": deleted_count}, "message": f"已删除 {deleted_count} 个技能"} + + +# ============================================================ +# 删除技能（单个） +# ============================================================ + +@router.delete("/{skill_id}") +async def delete_skill( + skill_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """删除技能（支持 EvolutionSkill 和 MarketApp 两种来源）""" + # 1) 先尝试 EvolutionSkill + skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() + if skill: + skill_name = skill.skill_name + user_id = skill.user_id + + # 记录事件 + ev = EvolutionEvent( + id=str(uuid.uuid4()), + user_id=user_id, + skill_name=skill_name, + event_type="deleted", + description=f"管理员 {admin_user.username} 删除技能 {skill_name}", + ) + db.add(ev) + + # 删除版本历史 + db.query(SkillVersion).filter(SkillVersion.skill_name == skill_name).delete() + + # 删除发布记录 + db.query(SkillPublishRecord).filter(SkillPublishRecord.skill_id == skill_id).delete() + + # 删除团队共享记录 + db.query(TeamSharedSkill).filter(TeamSharedSkill.skill_id == skill_id).delete() + + # 同步清理市场应用表中的同名技能 + market_app = db.query(MarketApp).filter( + MarketApp.type == "skill", + MarketApp.name == skill_name, + ).first() + if market_app: + db.query(AppInstallation).filter(AppInstallation.app_id == market_app.id).delete() + db.query(AppPurchase).filter(AppPurchase.app_id == market_app.id).delete() + db.query(AppReview).filter(AppReview.app_id == market_app.id).delete() + db.delete(market_app) + + db.delete(skill) + db.commit() + + LogService.log_operation( + user_id=admin_user.id, + username=admin_user.username, + action="delete_skill", + target="evolution_skills", + detail=f"删除技能 {skill_name}", + ) + + return {"code": 0, "message": f"技能 {skill_name} 已删除"} + + # 2) 再尝试 MarketApp 中导入的技能（type=skill） + market_app = db.query(MarketApp).filter( + MarketApp.type == "skill", + MarketApp.id == skill_id, + ).first() + if market_app: + skill_name = market_app.name + # 清理关联记录 + db.query(AppInstallation).filter(AppInstallation.app_id == market_app.id).delete() + db.query(AppPurchase).filter(AppPurchase.app_id == market_app.id).delete() + db.query(AppReview).filter(AppReview.app_id == market_app.id).delete() + db.delete(market_app) + db.commit() + + LogService.log_operation( + user_id=admin_user.id, + username=admin_user.username, + action="delete_skill", + target="market_apps", + detail=f"删除市场技能 {skill_name}", + ) + + return {"code": 0, "message": f"技能 {skill_name} 已删除"} + + # 3) 都未找到 + raise HTTPException(status_code=404, detail="技能不存在") + + +# ============================================================ +# 修改技能状态 +# ============================================================ + +class ChangeStateSchema(BaseModel): + state: str = Field(..., description="目标状态: baby/main/staging/frozen") + + +@router.put("/{skill_id}/state") +async def change_skill_state( + skill_id: str, + req: ChangeStateSchema, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """修改技能状态""" + if req.state not in SKILL_STATES: + raise HTTPException(status_code=400, detail=f"无效状态，可选值: {list(SKILL_STATES.keys())}") + + skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() + if not skill: + raise HTTPException(status_code=404, detail="技能不存在") + + old_state = skill.state + skill.state = req.state + skill.updated_at = datetime.now() + + # 记录状态变更事件 + ev_type = "promoted" if req.state in ("main", "staging") else "frozen" if req.state == "frozen" else "unfrozen" + ev = EvolutionEvent( + id=str(uuid.uuid4()), + user_id=skill.user_id, + skill_name=skill.skill_name, + event_type=ev_type, + description=f"管理员 {admin_user.username} 将技能状态从 {old_state} 变更为 {req.state}", + ) + db.add(ev) + db.commit() + db.refresh(skill) + + LogService.log_operation( + user_id=admin_user.id, + username=admin_user.username, + action="change_skill_state", + target="evolution_skills", + detail=f"修改技能 {skill.skill_name} 状态: {old_state} → {req.state}", + ) + + return {"code": 0, "data": skill_to_dict(skill)} + + +# ============================================================ +# 获取版本历史 +# ============================================================ + +@router.get("/{skill_id}/versions") +async def get_skill_versions( + skill_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取技能版本历史""" + skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() + if not skill: + raise HTTPException(status_code=404, detail="技能不存在") + + versions = db.query(SkillVersion).filter( + SkillVersion.skill_name == skill.skill_name, + SkillVersion.user_id == skill.user_id, + ).order_by(desc(SkillVersion.version)).all() + + return { + "code": 0, + "data": { + "skill_id": skill.id, + "skill_name": skill.skill_name, + "current_version": skill.version, + "versions": [version_to_dict(v) for v in versions], + } + } + + +# ============================================================ +# 回滚到指定版本 +# ============================================================ + +class RollbackSchema(BaseModel): + version: int = Field(..., ge=1, description="目标版本号") + + +@router.post("/{skill_id}/rollback") +async def rollback_skill_version( + skill_id: str, + req: RollbackSchema, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """回滚技能到指定版本""" + skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() + if not skill: + raise HTTPException(status_code=404, detail="技能不存在") + + target_version = db.query(SkillVersion).filter( + SkillVersion.skill_name == skill.skill_name, + SkillVersion.user_id == skill.user_id, + SkillVersion.version == req.version, + ).first() + + if not target_version: + raise HTTPException(status_code=404, detail=f"版本 {req.version} 不存在") + + old_version = skill.version + + # 更新 SKILL.md 文件 + if target_version.skill_md_content: + if skill.skill_md_path: + try: + with open(skill.skill_md_path, 'w', encoding='utf-8') as f: + f.write(target_version.skill_md_content) + except Exception as e: + logger.warning(f"回滚保存 SKILL.md 失败: {e}") + + # 记录新版本（基于目标版本） + new_version = old_version + 1 + sv = SkillVersion( + id=str(uuid.uuid4()), + user_id=skill.user_id, + skill_name=skill.skill_name, + version=new_version, + skill_md_content=target_version.skill_md_content, + git_commit_sha=None, + description=f"回滚到版本 {req.version}（管理员操作）", + ) + db.add(sv) + + skill.version = new_version + skill.updated_at = datetime.now() + + ev = EvolutionEvent( + id=str(uuid.uuid4()), + user_id=skill.user_id, + skill_name=skill.skill_name, + event_type="edited", + description=f"管理员 {admin_user.username} 回滚技能 {skill.skill_name} 从版本 {old_version} 到 {req.version}（新版本 {new_version}）", + version_before=old_version, + version_after=new_version, + ) + db.add(ev) + + db.commit() + db.refresh(skill) + + LogService.log_operation( + user_id=admin_user.id, + username=admin_user.username, + action="rollback_skill", + target="evolution_skills", + detail=f"回滚技能 {skill.skill_name} 到版本 {req.version}（新版本 {new_version}）", + ) + + return {"code": 0, "data": skill_to_dict(skill, include_md=True)} + + +# ============================================================ +# 发布到应用市场 +# ============================================================ + +class PublishSkillSchema(BaseModel): + price: float = Field(0.0, ge=0, description="定价（0表示免费）") + market_description: str = Field("", description="市场描述") + + +@router.post("/{skill_id}/publish") +async def publish_skill( + skill_id: str, + req: PublishSkillSchema, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """发布技能到应用市场""" + skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() + if not skill: + raise HTTPException(status_code=404, detail="技能不存在") + + # 检查是否已发布 + existing = db.query(SkillPublishRecord).filter( + SkillPublishRecord.skill_id == skill_id, + SkillPublishRecord.status == "published" + ).first() + + if existing: + # 更新现有发布 + existing.price = req.price + existing.market_description = req.market_description + msg = "更新发布信息" + else: + # 创建发布记录 + record = SkillPublishRecord( + id=str(uuid.uuid4()), + skill_id=skill_id, + skill_name=skill.skill_name, + user_id=skill.user_id, + price=req.price, + market_description=req.market_description, + status="published", + ) + db.add(record) + msg = "发布成功" + + db.commit() + + LogService.log_operation( + user_id=admin_user.id, + username=admin_user.username, + action="publish_skill", + target="skill_publish_records", + detail=f"{msg}: {skill.skill_name}，定价: ¥{req.price}", + ) + + return {"code": 0, "message": msg, "data": {"skill_id": skill_id, "price": req.price}} + + +# ============================================================ +# 下架技能 +# ============================================================ + +@router.post("/{skill_id}/unpublish") +async def unpublish_skill( + skill_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """从应用市场下架技能""" + skill = db.query(EvolutionSkill).filter(EvolutionSkill.id == skill_id).first() + if not skill: + raise HTTPException(status_code=404, detail="技能不存在") + + updated = db.query(SkillPublishRecord).filter( + SkillPublishRecord.skill_id == skill_id, + SkillPublishRecord.status == "published" + ).update({"status": "unpublished"}) + db.commit() + + if updated == 0: + raise HTTPException(status_code=400, detail="该技能未发布到市场") + + LogService.log_operation( + user_id=admin_user.id, + username=admin_user.username, + action="unpublish_skill", + target="skill_publish_records", + detail=f"下架技能: {skill.skill_name}", + ) + + return {"code": 0, "message": f"技能 {skill.skill_name} 已下架"} + + +# ============================================================ +# 获取统计数据 +# ============================================================ + +@router.get("/stats/overview") +async def get_skills_stats( + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取技能统计概览（EvolutionSkill + MarketApp type=skill 合并统计）""" + # EvolutionSkill 维度 + evo_total = db.query(EvolutionSkill).count() + evo_by_state = dict( + db.query(EvolutionSkill.state, func.count(EvolutionSkill.id)) + .group_by(EvolutionSkill.state).all() + ) + evo_by_category = dict( + db.query(EvolutionSkill.category, func.count(EvolutionSkill.id)) + .filter(EvolutionSkill.category.isnot(None)) + .group_by(EvolutionSkill.category).all() + ) + + # MarketApp(type=skill) 维度：状态映射 published→main, 其它→baby + market_total = db.query(MarketApp).filter(MarketApp.type == "skill").count() + market_rows = db.query(MarketApp.status, func.count(MarketApp.id)).filter( + MarketApp.type == "skill" + ).group_by(MarketApp.status).all() + market_published = sum(cnt for status, cnt in market_rows if (status or "") == "published") + # 各 category 计数（market 表 category 在 SQL 列有，ORM 无此属性，用 SQL 拿） + market_by_category = {} + try: + market_by_category_rows = db.execute( + text("SELECT capability, COUNT(*) FROM market_apps WHERE type='skill' AND capability IS NOT NULL GROUP BY capability") + ).fetchall() + market_by_category = {cat or "other": cnt for cat, cnt in market_by_category_rows} + except Exception: + pass + + # 合并 + total_skills = evo_total + market_total + by_state = {k: evo_by_state.get(k, 0) for k in SKILL_STATES} + by_state["main"] = by_state.get("main", 0) + market_published + by_state["baby"] = by_state.get("baby", 0) + (market_total - market_published) + by_category = {} + for c in (evo_by_category or {}).items(): + by_category[c[0]] = by_category.get(c[0], 0) + c[1] + for c in (market_by_category or {}).items(): + by_category[c[0]] = by_category.get(c[0], 0) + c[1] + + # 原始发布记录 + market 已发布 = 全部已发布 + skill_record_published = db.query(SkillPublishRecord).filter( + SkillPublishRecord.status == "published" + ).count() + total_published = skill_record_published + market_published + total_versions = db.query(SkillVersion).count() + + return { + "code": 0, + "data": { + "total_skills": total_skills, + "by_state": by_state, + "by_category": by_category, + "total_published": total_published, + "total_versions": total_versions, + } + } diff --git a/backend/python/app/api/chat_context.py b/backend/python/app/api/chat_context.py index 4c378ac..23562d8 100644 --- a/backend/python/app/api/chat_context.py +++ b/backend/python/app/api/chat_context.py @@ -1121,7 +1121,7 @@ def _build_system_prompt( all_mcp: list = None, - + db: Session = None, ) -> tuple: @@ -1195,7 +1195,7 @@ def _build_system_prompt( - pk = get_platform_knowledge() + pk = get_platform_knowledge(db=db) diff --git a/backend/python/app/api/chat_media.py b/backend/python/app/api/chat_media.py index 6b3d723..fd11bbf 100644 --- a/backend/python/app/api/chat_media.py +++ b/backend/python/app/api/chat_media.py @@ -1566,11 +1566,16 @@ async def _generate_media_task(ai_msg_id: str, ai_model_id: str, prompt: str, us except Exception as e: - - - - logger.warning(f"[_generate_media_task] 后台生成异常: {e}") - + logger.warning(f"[_generate_media_task] 后台生成异常: {e}", exc_info=True) + # 兜底回写：任何意外异常都不能让消息永久停在「⏳ 生成中…」 + try: + db.rollback() + _err = db.query(Message).filter(Message.id == ai_msg_id).first() + if _err and str(_err.content or "").startswith("⏳"): + _err.content = "⚠️ 媒体生成失败，请稍后重试" + db.commit() + except Exception: + pass finally: diff --git a/backend/python/app/api/chat_service.py b/backend/python/app/api/chat_service.py index e24614b..de3ed79 100644 --- a/backend/python/app/api/chat_service.py +++ b/backend/python/app/api/chat_service.py @@ -1,4273 +1,4304 @@ -# Auto-extracted from chat.py during Phase 1 refactor (Strangler Fig). - -# Source of truth: scripts/phase1_extract.py. Do not hand-merge back. - - - -from fastapi import APIRouter, Depends, HTTPException, Query, Response - -from sqlalchemy.orm import Session - -from sqlalchemy import and_, desc, asc, func - -from pydantic import BaseModel, Field - -from typing import Optional - -import uuid - -from datetime import datetime, timezone, timedelta - -import re - -import json - -import asyncio - -import math - -import httpx - -import os - -import logging - -from ..database import get_db, SessionLocal - -from app.core.observability import span, get_trace_id - -from ..models import ( - - - - - - - - User, Conversation, Message, AIModel, ModelProvider, TokenUsageLog, - - - - - - - - KnowledgeBase, KnowledgeDocument, KnowledgeDocumentChunk, EvolutionSkill, - - - - - - - - EvolutionTrajectory, MarketApp, AppPurchase, AppInstallation, - - - - - - - - MCPServer, - - - - - - - - gen_uuid - - - - - - - -) - -from ..auth import get_current_user - -from ..platform_knowledge import get_platform_knowledge - -from ..mcp_client import list_mcp_tools, call_mcp_tool - -from ..middleware.log_service import LogService - -from ..capability_registry import resolve_auto_model - -from app.api.chat_agents import (_call_external_agent) - -from app.api.chat_context import (_build_system_prompt, _get_conversation_context, _ingest_answer_to_dify, _retrieve_kb_context, _build_chat_history, _maybe_compress_history) - -from app.api.chat_mcp import (_list_enabled_mcp_servers, _resolve_active_mcp_servers) - -from app.api.chat_media import (_generate_media_task, detect_ambiguous_creative_intent, detect_media_intent) - -from app.api.chat_skill import (_discover_installed_skills, _execute_skill_tool, _resolve_market_agents, _resolve_market_skills) - -from app.api.chat_utils import (SendMessageRequest, _proxy_for_url, cst_now, logger) - - - -async def _send_message_core( - - - - - - - - conv_id: str, - - - - - - - - req: SendMessageRequest, - - - - - - - - db: Session = Depends(get_db), - - - - - - - - current_user: User = Depends(get_current_user), - - - - - - - -): - - - - - - - - conv = db.query(Conversation).filter( - - - - - - - - and_(Conversation.id == conv_id, Conversation.user_id == current_user.id) - - - - - - - - ).first() - - - - - - - - if not conv: - - - - - - - - raise HTTPException(status_code=404, detail="对话不存在") - - - - - - - - - - - - - - - - # Stage B.2: 多轮上下文保持 — 如果当前请求没有skill/agent/kb, - - - - - - - - # 从对话历史中继承上一次的选择 - - - - - - - - effective_skill_ids = req.skill_ids or [] - - - - - - - - effective_agent_ids = req.agent_ids or [] - - - - - - - - effective_kb_ids = req.knowledge_base_ids or [] - - - - - - - - effective_mcp_ids = req.mcp_ids or [] - - - - - - - - effective_use_tools = req.use_tools or False - - - - - - - - - - - - - - - - if not effective_skill_ids and not effective_agent_ids and not effective_kb_ids and not effective_mcp_ids: - - - - - - - - # 从对话上下文继承 - - - - - - - - prev_context = _get_conversation_context(db, conv_id) - - - - - - - - if prev_context: - - - - - - - - effective_skill_ids = prev_context.get("skill_ids", []) - - - - - - - - effective_agent_ids = prev_context.get("agent_ids", []) - - - - - - - - effective_kb_ids = prev_context.get("knowledge_base_ids", []) - - - - - - - - effective_mcp_ids = prev_context.get("mcp_ids", []) - - - - - - - - if effective_skill_ids or effective_agent_ids or effective_mcp_ids: - - - - - - - - effective_use_tools = True - - - - - - - - - - - - - - - - # 附件内容拼接到用户消息中(文本类文件附带内容, 供模型参考) - - - - - - - - attachment_block = "" - - - - - - - - if req.attachments: - - - - - - - - parts = [] - - - - - - - - for att in req.attachments: - - - - - - - - name = att.get("name", "附件") if isinstance(att, dict) else str(att) - - - - - - - - ftype = att.get("file_type", "") if isinstance(att, dict) else "" - - - - - - - - content = att.get("content", "") if isinstance(att, dict) else "" - - - - - - - - if content: - - - - - - - - parts.append(f"【附件: {name}】\n{content}") - - - - - - - - else: - - - - - - - - parts.append(f"【附件(二进制, 名称: {name}, 类型: {ftype})】") - - - - - - - - if parts: - - - - - - - - attachment_block = "\n\n" + "\n\n".join(parts) - - - - - - - - - - - - - - - - # 保存用户消息(含skill/agent/kb/mcp选择用于上下文保持) - - - - - - - - user_msg = Message( - - - - - - - - id=gen_uuid(), - - - - - - - - conversation_id=conv_id, - - - - - - - - role="user", - - - - - - - - content=req.content + attachment_block, - - - - - - - - token_used=len(req.content) // 2, - - - - - - - - model_id=req.model_id or conv.model_id, - - - - - - - - skill_ids=req.skill_ids if req.skill_ids else None, - - - - - - - - agent_ids=req.agent_ids if req.agent_ids else None, - - - - - - - - knowledge_ids=req.knowledge_base_ids if req.knowledge_base_ids else None, - - - - - - - - mcp_ids=req.mcp_ids if req.mcp_ids else None, - - - - - - - - attachments=req.attachments if req.attachments else None, - - - - - - - - created_at=cst_now(), - - - - - - - - ) - - - - - - - - db.add(user_msg) - - - - - - - - - - - - - - - - # ===== 模糊创作意图拦截 ===== - - - - - - - - # 用户使用创作类动词(制作/创作/做/生成…)但未明确产出形式时，先与用户确认方向， - - - - - - - - # 避免直接调用大模型被内容安全策略拒答(如返回 "high risk")。 - - - - - - - - clarify = detect_ambiguous_creative_intent(req.content) - - - - - - - - if clarify: - - - - - - - - clarify_content = "<>" + json.dumps( - - - - - - - - {"prompt": req.content, "options": clarify["options"]}, ensure_ascii=False - - - - - - - - ) + "<>" - - - - - - - - ai_msg = Message( - - - - - - - - id=gen_uuid(), - - - - - - - - conversation_id=conv_id, - - - - - - - - role="assistant", - - - - - - - - content=clarify_content, - - - - - - - - token_used=0, - - - - - - - - model_id=req.model_id or conv.model_id, - - - - - - - - created_at=cst_now(), - - - - - - - - ) - - - - - - - - db.add(ai_msg) - - - - - - - - db.commit() - - - - - - - - db.refresh(user_msg) - - - - - - - - db.refresh(ai_msg) - - - - - - - - return { - - - - - - - - "code": 0, - - - - - - - - "data": { - - - - - - - - "user_message": { - - - - - - - - "id": user_msg.id, "role": user_msg.role, "content": user_msg.content, - - - - - - - - "token_used": user_msg.token_used, "created_at": str(user_msg.created_at), - - - - - - - - }, - - - - - - - - "assistant_message": { - - - - - - - - "id": ai_msg.id, "role": ai_msg.role, "content": clarify_content, - - - - - - - - "created_at": str(ai_msg.created_at), "clarify": clarify, - - - - - - - - }, - - - - - - - - }, - - - - - - - - "message": "请选择创作方向", - - - - - - - - } - - - - - - - - - - - - - - - - # ===== 核心聊天引擎 ===== - - - - - - - - user_id = current_user.id - - - - - - - - model_id = req.model_id or conv.model_id - - - - - - - - prompt_tokens = 0 - - - - - - - - completion_tokens = 0 - - - - - - - - total_tokens = 0 - - - - - - - - ai_content = "" - - - - - - - - - - - - - - - - ai_model = None - - - - - - - - skills = [] # 初始化避免UnboundLocalError - process_steps = [] # 过程步骤(思考/技能/MCP执行)，用于前端收拢式展示 - - - - - - - - - agents = [] # 初始化避免UnboundLocalError - - - - - - - - media_generating = False # 提前初始化：外部Agent分支与末尾均会引用 - - - - - - - - external_handled = False # 是否已由外部Agent(Dify/n8n/Coze)远程处理 - - - - - - - - - - - - - - - - # ===== 外部 Agent 远程调用分支 (Dify / n8n / Coze) ===== - - - - - - - - # 若本轮对话关联了带有效 agent_endpoint 的外部 Agent，则直接远程调用其平台， - - - - - - - - # 跳过本地 LLM「扮演」逻辑。effective_agent_ids 已在前面从请求/上下文继承。 - - - - - - - - external_agents = _resolve_market_agents(db, effective_agent_ids, user_id) - - - - - - - - external_agent = next((a for a in external_agents if a.get("agent_endpoint")), None) - - - - # ===== 3.3 多 Agent 编排(图引擎)快路 ===== - - _orchestrator = next((a for a in external_agents if a.get("orchestration")), None) - - if _orchestrator: - - try: - - from app.core.orchestrator import run_graph - - _answer, _trace = await run_graph( - - graph=_orchestrator["orchestration"], - - query=req.content + attachment_block, - - db=db, - - user_id=user_id, - - conv_id=conv_id, - - trace_id=get_trace_id(), - - ) - - ai_content = _answer or "编排未产生内容。" - - logger.info( - - f"[orchestrator] 图执行完成 trace_id={get_trace_id()} nodes={len(_trace.get('nodes', []))}" - - ) - - except Exception as e: - - logger.warning(f"[orchestrator] 图执行失败: {e}") - - ai_content = f"⚠️ 多 Agent 编排执行失败：{str(e)[:200]}" - - external_handled = True - - external_agent = None # 阻止下方外部 Agent 分支重复执行 - - - - - - - - if external_agent: - - - - - - - - agents = external_agents - - - - - - - - # 组装最近历史(供 n8n 等无平台会话态的服务维持多轮上下文) - - - - - - - - _history = _build_chat_history(db, conv_id, max_tokens=4000, exclude_latest_user=True) - - - - - - - - try: - - - - - - - - answer, new_ext_id = await _call_external_agent( - - - - - - - - db, external_agent, - - - - - - - - query=req.content + attachment_block, - - - - - - - - user_id=user_id, - - - - - - - - ext_conversation_id=conv.ext_agent_conversation_id, - - - - - - - - history=_history, - - - - - - - - ) - - - - - - - - ai_content = answer or "外部 Agent 未返回内容。" - - - - - - - - if new_ext_id: - - - - - - - - conv.ext_agent_conversation_id = new_ext_id - - - - - - - - except Exception as e: - - - - - - - - logger.warning( - - - - - - - - f"[external_agent] 远程调用失败 platform={external_agent.get('platform')} " - - - - - - - - f"endpoint={external_agent.get('agent_endpoint')}: {e}" - - - - - - - - ) - - - - - - - - ai_content = f"⚠️ 外部 Agent 调用失败：{str(e)[:200]}" - - - - - - - - external_handled = True - - - - - - - - - - - - - - - - # 解析模型：支持 Auto 模式（model_id='auto' 或未指定时，按能力自动匹配平台默认模型） - - - - - - - - if not external_handled: - - - - - - - - if model_id and model_id != "auto": - - - - - - - - ai_model = db.query(AIModel).filter(AIModel.id == model_id).first() - - - - - - - - if not ai_model: - - - - - - - - # Auto / 未指定 → 按用户输入意图匹配对应能力的默认模型，否则回退文本对话 - - - - - - - - intent = detect_media_intent(req.content) - - - - - - - - resolved = resolve_auto_model(intent, db) if intent else None - - - - - - - - ai_model = resolved or resolve_auto_model("text", db) - - - - - - - - - - - - - - - - if ai_model: - - - - - - - - # ====== 图像/视频生成模型分支 ====== - - - - - - - - caps = [c.lower() for c in (ai_model.capabilities or [])] if ai_model else [] - - - - - - - - media_generating = False - - - - - - - - if ai_model and ("image" in caps or "video" in caps): - - - - - - - - # 异步生成：立即返回占位内容，后台任务轮询 AGNES 并回写， - - - - - - - - # 避免最长 5 分钟的视频轮询阻塞 HTTP 请求导致客户端超时断开。 - - - - - - - - media_generating = True - - - - - - - - if "video" in caps: - - - - - - - - ai_content = "⏳ 视频生成中…（通常 1~3 分钟，完成后自动刷新）" - - - - - - - - else: - - - - - - - - ai_content = "⏳ 图像生成中…（完成后自动刷新）" - - - - - - - - elif ai_model and ai_model.endpoint: - - - - - - - - provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first() - - - - - - - - api_key = ai_model.api_key or (provider.api_key if provider else None) - - - - - - - - api_url = ai_model.endpoint - - - - - - - - - - - - - - - - if not api_url and provider and provider.api_base_url: - - - - - - - - api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions" - - - - - - - - - - - - - - - - if api_key and api_url: - - - - - - - - try: - - - - - - - - # Stage A+B: 解析Skills和Agents - - - - - - - - skills = _resolve_market_skills(db, effective_skill_ids, user_id) - - - - - - - - agents = _resolve_market_agents(db, effective_agent_ids, user_id) - - - - - - - - mcps = _resolve_active_mcp_servers(db, effective_mcp_ids) - - - - - - - - # MCP 工具调用链路：拉取各 MCP 工具并转为 function definitions - - - - mcp_servers_conn = mcps - - - - mcp_function_definitions = [] - - - - mcp_tool_map = {} - - - - mcp_idx = 0 - - - - for _srv in mcp_servers_conn: - - - - try: - - - - async with span("mcp.list_tools", server=_srv.get("name") if isinstance(_srv, dict) else None): - - _tools = await list_mcp_tools(_srv) - - - - except Exception as _e: - - - - logger.warning(f"MCP 拉取工具失败 [{_srv.get('name')}]: {_e}") - - - - _tools = [] - - - - if _tools: - - - - try: - - - - _upd = db.query(MCPServer).filter(MCPServer.id == _srv["id"]).first() - - - - if _upd: - - - - _upd.tools_count = len(_tools) - - - - db.commit() - - - - except Exception: - - - - pass - - - - for _t in _tools: - - - - _fname = f"mcp_tool_{mcp_idx}" - - - - mcp_tool_map[_fname] = { - - - - "server_id": _srv["id"], - - - - "name": _t.get("name"), - - - - "server_name": _srv["name"], - - - - } - - - - _schema = _t.get("inputSchema") or {"type": "object", "properties": {}} - - - - if not isinstance(_schema, dict) or _schema.get("type") != "object": - - - - _schema = {"type": "object", "properties": {}} - - - - mcp_function_definitions.append({ - - - - "name": _fname, - - - - "description": f"[MCP·{_srv['name']}] {_t.get('description', '')}", - - - - "parameters": _schema, - - - - }) - - - - mcp_idx += 1 - - - - if mcp_function_definitions: - - - - effective_use_tools = True - - - - - - - - - - - - - - - - # Issue 8: 默认技能自动发现（仅当用户未显式选择技能时启用） - - - - - - - - # 每轮先寻找本账号已安装且平台已部署的技能，多个适用时由模型自行判断选 1 个最优。 - - - - - - - - used_auto_skills = False - - - - - - - - if not effective_skill_ids: - - - - - - - - try: - - - - - - - - discovered = _discover_installed_skills(db, user_id) - - - - - - - - except Exception as e: - - - - - - - - logger.warning(f"自动发现已安装技能失败: {e}") - - - - - - - - discovered = [] - - - - - - - - if discovered: - - - - - - - - skills = discovered - - - - - - - - used_auto_skills = True - - - - - - - - effective_use_tools = True - - - - - - - - - - - - - - - - # Stage B.1: Agent可能自带skill_ids和kb_ids - - - - - - - - agent_extra_skill_ids = [] - - - - - - - - agent_extra_kb_ids = [] - - - - - - - - agent_model_override = None - - - - - - - - for agent in agents: - - - - - - - - if agent.get("skill_ids"): - - - - - - - - agent_extra_skill_ids.extend(agent["skill_ids"]) - - - - - - - - if agent.get("knowledge_base_ids"): - - - - - - - - agent_extra_kb_ids.extend(agent["knowledge_base_ids"]) - - - - - - - - if agent.get("model_id"): - - - - - - - - agent_model_override = agent["model_id"] - - - - - - - - - - - - - - - - # 合并Agent自带的skills和KBs - - - - - - - - all_skill_ids = list(set(effective_skill_ids + agent_extra_skill_ids)) - - - - - - - - all_kb_ids = list(set(effective_kb_ids + agent_extra_kb_ids)) - - - - - - - - - - - - - - - - if agent_extra_skill_ids or agent_extra_kb_ids: - - - - - - - - # 重新解析合并后的skills - - - - - - - - skills = _resolve_market_skills(db, all_skill_ids, user_id) - - - - - - - - effective_use_tools = True - - - - - - - - - - - - - - - - # Stage C: 增强版KB检索 - - - - - - - - kb_context = await _retrieve_kb_context(db, all_kb_ids, req.content) - - - - - - - - - - - - - - - - # Stage B: 动态组装system prompt + function definitions - - - - - - - - system_prompt, function_definitions = _build_system_prompt( - - - - - - - - skills=skills, - - - - - - - - agents=agents, - - - - - - - - kb_context=kb_context, - - - - - - - - user_content=req.content, - - - - - - - - mcp=mcps, - - - - mcp_functions=mcp_function_definitions, - - - - auto_discovered=used_auto_skills, - - all_mcp=_list_enabled_mcp_servers(db), - - - - ) - - - - - - - - - - - - - - - - # 构建消息列表 - - - - - - - - # 实际调用追踪（Issue：反馈真实使用的技能/MCP，而非注入的全部） - - called_skill_names = [] - - called_mcp_names = [] - - - - messages = [{"role": "system", "content": system_prompt}] - - - - - - - - - - - - - - - - # 分层上下文引擎：预算感知历史 + 压缩摘要(若有) - - messages.extend(_build_chat_history(db, conv_id, max_tokens=6000, exclude_latest_user=True)) - - - - - - - - - - - - - - - - # Issue 1: 必须把附件内容(attachment_block)一并发给模型， - - - - - - - - # 否则用户在对话窗上传文件后模型“看不到”文件内容。 - - - - - - - - messages.append({"role": "user", "content": req.content + attachment_block}) - - - - - - - - - - - - - - - - # Stage B.3: Function Calling循环(最多5轮) - - - - - - - - max_tool_rounds = 5 - - - - - - - - # ===== 3.1 tool_policy 沙箱：聚合策略 + 初始化预算计数器 ===== - - _agent_tool_policy = _resolve_agent_tool_policy(agents) - - _tool_call_count = 0 - - for round_num in range(max_tool_rounds): - - _round_tool_count = 0 - - - - - - - - request_body = { - - - - - - - - "model": ai_model.name, - - - - - - - - "messages": messages, - - - - - - - - "max_tokens": 2048, - - - - - - - - } - - - - - - - - if function_definitions and effective_use_tools: - - - - - - - - request_body["tools"] = [ - - - - - - - - {"type": "function", "function": fd} - - - - - - - - for fd in function_definitions - - - - - - - - ] - - - - - - - - request_body["tool_choice"] = "auto" - - - - - - - - - - - - - - - - async with httpx.AsyncClient( - - - - - - - - timeout=ai_model.timeout or 120, - - - - - - - - proxy=_proxy_for_url(api_url), - - - - - - - - trust_env=False, - - - - - - - - ) as client: - - - - - - - - async with span("llm.completion", model=ai_model.id if ai_model else None): - - resp = await client.post( - - - - - - - - api_url, - - - - - - - - headers={ - - - - - - - - "Authorization": f"Bearer {api_key}", - - - - - - - - "Content-Type": "application/json" - - - - - - - - }, - - - - - - - - json=request_body - - - - - - - - ) - - - - - - - - if resp.status_code != 200: - - - - - - - - if round_num == 0: - - - - - - - - ai_content = f"AI模型调用失败 (HTTP {resp.status_code}), 请检查模型配置。" - - - - - - - - break - - - - - - - - - - - - - - - - data = resp.json() - - - - - - - - choice = data.get("choices", [{}])[0] - - - - - - - - message = choice.get("message", {}) - - - - - - - - - - - - - - - - # 记录token使用量 - - - - - - - - usage = data.get("usage", {}) - - - - - - - - prompt_tokens += usage.get("prompt_tokens", 0) - - - - - - - - completion_tokens += usage.get("completion_tokens", 0) - - - - - - - - total_tokens += usage.get("total_tokens", 0) - - - - - - - - - - - - - - - - # 检查是否有tool_calls - - - - - - - - tool_calls = message.get("tool_calls", []) - # 过程步骤：捕获模型思考/推理(产生工具调用前的推理) - _think = (message.get("content") or "").strip() - if _think and tool_calls: - process_steps.append({ - "type": "thinking", - "text": _think, - }) - - - - - - - - - if tool_calls and effective_use_tools: - - - - - - - - # 将assistant消息(含tool_calls)加入上下文 - - - - - - - - messages.append({ - - - - - - - - "role": "assistant", - - - - - - - - "content": message.get("content") or "", - - - - - - - - "tool_calls": tool_calls, - - - - - - - - }) - - - - - - - - - - - - - - - - # Stage A.2: 真实Skill执行 - - - - - - - - for tc in tool_calls: - - - - - - - - func_name = tc.get("function", {}).get("name", "") - - # ===== 3.1 tool_policy 沙箱：逐工具校验（白名单 + 预算 + 每轮上限） ===== - - _mcp_info = mcp_tool_map.get(func_name) if func_name.startswith("mcp_tool_") else None - - _policy_check = _check_tool_policy(func_name, _mcp_info, skills, _agent_tool_policy, _tool_call_count, _round_tool_count) - - if not _policy_check["allowed"]: - - _reason = _policy_check["reason"] - - logger.warning( - - f"[tool_policy] blocked tool call: agent={_agent_tool_policy.get('agent_id') if _agent_tool_policy else None} " - - f"tool={func_name} reason={_reason} trace={get_trace_id()}" - - ) - - tool_result = f"[工具调用已被策略拦截] {_policy_check.get('message', _reason)}" - - messages.append({ - - "role": "tool", - - "tool_call_id": tc.get("id", f"call_{round_num}"), - - "content": tool_result, - - }) - - _tool_call_count += 1 - - _round_tool_count += 1 - - continue - - _tool_call_count += 1 - - _round_tool_count += 1 - - - - - - - - func_args_str = tc.get("function", {}).get("arguments", "{}") - - - - - - - - try: - - - - - - - - func_args = json.loads(func_args_str) - - - - - - - - except Exception: - - - - - - - - func_args = {"input": func_args_str} - - - - - - - - - - - - - - - - if func_name.startswith("mcp_tool_"): - - - - # MCP 工具调用：代理到对应 MCP 服务执行 tools/call - - - - _info = mcp_tool_map.get(func_name) - - - - tool_result = "[MCP 工具未找到或映射缺失]" - - - - if _info: - - - - _srv = next( - - - - (s for s in mcp_servers_conn if s["id"] == _info["server_id"]), - - - - None, - - - - ) - - - - if _srv: - - - - async with span(f"mcp.call:{_info.get('name') if isinstance(_info, dict) else None}"): - - tool_result = await call_mcp_tool( - - - - _srv, _info["name"], func_args - - - - ) - - - - called_mcp_names.append(_srv.get("name", "")) - process_steps.append({ - "type": "mcp", - "name": _info.get("name") if isinstance(_info, dict) else func_name, - "server": _srv.get("name", ""), - "args": func_args, - "result": (tool_result or "")[:800], - }) - - - - - else: - - - - async with span(f"skill.execute:{func_name}"): - - tool_result = _execute_skill_tool( - - - - db, func_name, func_args, skills, req.content - - - - ) - - - - _sk = next((s for s in skills if s.get("func_name") == func_name), None) - - - - if _sk: - - - - called_skill_names.append(_sk.get("name", "")) - process_steps.append({ - "type": "skill", - "name": (_sk.get("name") if _sk else func_name), - "args": func_args, - "result": (tool_result or "")[:800], - }) - - - - - - - - - messages.append({ - - - - - - - - "role": "tool", - - - - - - - - "tool_call_id": tc.get("id", f"call_{round_num}"), - - - - - - - - "content": tool_result, - - - - - - - - }) - - - - - - - - - - - - - - - - # 继续下一轮让模型处理tool结果 - - - - - - - - continue - - - - - - - - else: - - - - - - - - # 普通回复 - - - - - - - - ai_content = message.get("content", "") - - - - - - - - if not ai_content and round_num > 0: - - - - - - - - ai_content = "AI模型已处理工具调用结果, 但返回了空响应。" - - - - - - - - break - - - - - - - - - - - - - - - - if not ai_content: - - - - - - - - ai_content = "AI模型返回了空响应, 请稍后重试。" - - - - - - - - - - - - - - - - except Exception as e: - - - - - - - - logger.warning(f"AI API调用异常: {e}") - - - - - - - - ai_content = "AI服务暂时不可用, 请稍后重试或检查模型API配置。" - - - - - - - - else: - - - - - - - - ai_content = "该模型缺少API密钥或端点配置, 请在后台管理-大模型库中配置API Key和Endpoint。" - - - - - - - - else: - - - - - - - - ai_content = "请选择一个已配置的AI模型进行对话。" - - - - - - - - elif not external_handled: - - - - - - - - ai_content = "请先在后台管理-大模型库配置默认对话模型（或聊天中选择模型），Auto 模式下将自动使用平台默认模型。" - - - - - - - - - - - - - - - - ai_token_count = len(ai_content) // 2 - - - - - - - - ai_msg = Message( - - - - - - - - id=gen_uuid(), - - - - - - - - conversation_id=conv_id, - - - - - - - - role="assistant", - - - - - - - - content=ai_content, - - - - - - - - token_used=ai_token_count, - - - - - - - - model_id=ai_model.id if ai_model else (req.model_id or conv.model_id), - - - - - - - - skill_ids=req.skill_ids if req.skill_ids else None, - - - - - - - - agent_ids=req.agent_ids if req.agent_ids else None, - - - - - - - - knowledge_ids=req.knowledge_base_ids if req.knowledge_base_ids else None, - - - - - - - - created_at=cst_now(), - - - - - - - - ) - - - - - - - - db.add(ai_msg) - - - - - - - - - - - - - - - - # 后台异步：将实质性文本回答入库到 Dify 知识库（量化后可供 RAG 复用） - - - - - - - - _ingest_text = (ai_content or "").strip() - - - - - - - - if len(_ingest_text) > 120 and not ( - - - - - - - - _ingest_text.startswith("[MEDIA") - - - - - - - - or _ingest_text.startswith("data:") - - - - - - - - or _ingest_text.startswith("http") - - - - - - - - or " 反馈配对的技能名 - - - - - - - - try: - - - - - - - - _sel = _resolve_market_skills(db, effective_skill_ids, user_id) - - - - - - - - used_skills = [s["name"] for s in _sel] - - - - - - - - except Exception: - - - - - - - - used_skills = [] - - - - - - - - elif called_skill_names: - - - - - - - - # 未显式选技能 -> 反馈 AI 本轮实际调用的技能（可能为空） - - - - - - - - used_skills = list(dict.fromkeys(called_skill_names)) - - - - - - - - if effective_mcp_ids: - - - - - - - - # 用户显式选择了 MCP -> 反馈配对的 MCP 名 - - - - - - - - try: - - - - - - - - _sel_mcp = _resolve_active_mcp_servers(db, effective_mcp_ids) - - - - - - - - used_mcp = [m["name"] for m in _sel_mcp] - - - - - - - - except Exception: - - - - - - - - used_mcp = [] - - - - - - - - elif called_mcp_names: - - - - - - - - used_mcp = list(dict.fromkeys(called_mcp_names)) - - - - - - - - ai_msg.used_skills = used_skills - - - - - - - - ai_msg.used_mcp = used_mcp - - - - - - - - - - - - - - - - # 更新对话统计 - - - - - - - - conv.knowledge_graph_count = (conv.knowledge_graph_count or 0) + 1 - - - - - - - - conv.updated_at = cst_now() - - - - - - - - - - - - - - - - # 记录Token使用（媒体生成走 _generate_media_task 异步补写真实消耗，此处跳过避免占位文本重复记账） - - - - - - - - if not media_generating: - - - - - - - - token_usage = TokenUsageLog( - - - - - - - - id=gen_uuid(), - - - - - - - - user_id=current_user.id, - - - - - - - - model_id=ai_model.id if ai_model else (req.model_id or conv.model_id), - - - - - - - - model_name=ai_model.name if ai_model else None, - - - - - - - - conversation_id=conv_id, - - trace_id=get_trace_id(), - - - - - - - - - - prompt_tokens=prompt_tokens or len(req.content) // 2, - - - - - - - - completion_tokens=completion_tokens or ai_token_count, - - - - - - - - total_tokens=total_tokens or (len(req.content) // 2 + ai_token_count), - - - - - - - - created_at=cst_now(), - - - - - - - - ) - - - - - - - - db.add(token_usage) - - - - - - - - - - - - - - - - # 进化轨迹记录 - - - - - - - - try: - - - - - - - - existing = db.query(EvolutionTrajectory).filter( - - - - - - - - EvolutionTrajectory.conversation_id == conv_id - - - - - - - - ).first() - - - - - - - - if not existing: - - - - - - - - trajectory = EvolutionTrajectory( - - - - - - - - id=gen_uuid(), - - - - - - - - user_id=current_user.id, - - - - - - - - conversation_id=conv_id, - - - - - - - - status="discovered", - - - - - - - - token_used=ai_token_count, - - - - - - - - created_at=cst_now(), - - - - - - - - ) - - - - - - - - db.add(trajectory) - - - - - - - - db.flush() - - - - - - - - - - - - - - - - try: - - - - - - - - from app.services.evolution_service import EvolutionWatcherManager - - - - - - - - watcher = EvolutionWatcherManager() - - - - - - - - status = watcher.get_user_status(current_user.id, lambda: SessionLocal()) - - - - - - - - if not status.get("scheduler_running"): - - - - - - - - watcher.start_user_evolution(current_user.id, lambda: SessionLocal()) - - - - - - - - logger.info(f"[Evolution] 自动启动用户 {current_user.id} 的进化Pipeline") - - - - - - - - except Exception as evolution_err: - - - - - - - - logger.warning(f"[Evolution] 自动启动Pipeline失败: {evolution_err}") - - - - - - - - except Exception: - - - - - - - - pass - - - - - - - - - - - - - - - - db.commit() - - - - - - - - # 后台异步压缩历史(分层记忆)，不阻塞本次响应 - - asyncio.create_task(_maybe_compress_history(conv_id)) - - - - db.refresh(user_msg) - - - - - - - - db.refresh(ai_msg) - - - - - - - - - - - - - - - - # 媒体生成改为后台异步：立即返回占位，后台任务完成后回写消息内容 - - - - - - - - if media_generating: - - - - - - - - try: - - - - - - - - asyncio.create_task( - - - - - - - - _generate_media_task(ai_msg.id, ai_model.id, req.content, current_user.id) - - - - - - - - ) - - - - - - - - except Exception as e: - - - - - - - - logger.warning(f"[_generate_media_task] 启动后台任务失败: {e}") - - - - - - - - - - - - - - - - return { - - - - - - - - "code": 0, - - - - - - - - "data": { - - - - - - - - "user_message": { - - - - - - - - "id": user_msg.id, "role": user_msg.role, "content": user_msg.content, - - - - - - - - "token_used": user_msg.token_used, "created_at": str(user_msg.created_at), - - - - - - - - }, - - - - - - - - "assistant_message": { - - - - - - - - "id": ai_msg.id, "role": ai_msg.role, "content": ai_msg.content, - - - - - - - - "token_used": ai_msg.token_used, "created_at": str(ai_msg.created_at), - - - - - - - - "used_skills": used_skills, - - "used_mcp": used_mcp, - - - - - - - - "generating": media_generating, - "process_steps": process_steps, - - - - - - - - - }, - - - - - - - - }, - - - - - - - - "message": "发送成功", - - - - - - - - } - - - - - -# ====================================================================== - -# 3.1 tool_policy 沙箱辅助函数 - -# ====================================================================== - -def _resolve_agent_tool_policy(agents): - - """从已解析的 agents 聚合工具策略。多 agent 时取首个带有非空策略者。""" - - if not agents: - - return None - - for ag in agents: - - tp = ag.get("tool_policy") - - if tp and isinstance(tp, dict): - - tp = dict(tp) - - tp["agent_id"] = ag.get("id") - - return tp - - return None - - - - - -def _check_tool_policy(func_name, mcp_info, skills, policy, call_count, round_count): - - """校验单个工具调用是否被策略允许。 - - - - 返回 {"allowed": bool, "reason": str, "tool_id": str|None, "message": str} - - policy 为 None/空 -> 允许（向后兼容）。 - - """ - - if not policy: - - return {"allowed": True, "tool_id": None} - - allowed = policy.get("allowed_tools") or {} - - skill_ids = allowed.get("skill_ids") or [] - - mcp_ids = allowed.get("mcp_ids") or [] - - is_mcp = func_name.startswith("mcp_tool_") - - tool_id = None - - if is_mcp: - - if mcp_info and isinstance(mcp_info, dict): - - tool_id = mcp_info.get("server_id") - - if mcp_ids and tool_id not in mcp_ids: - - return {"allowed": False, "reason": "mcp_not_in_allowlist", "tool_id": tool_id, - - "message": "MCP 服务不在该智能体的允许列表中"} - - else: - - if skills: - - _s = next((s for s in skills if s.get("func_name") == func_name), None) - - if _s: - - tool_id = _s.get("id") - - if skill_ids and tool_id not in skill_ids: - - return {"allowed": False, "reason": "skill_not_in_allowlist", "tool_id": tool_id, - - "message": "技能不在该智能体的允许列表中"} - - budget = policy.get("max_calls_per_conversation") - - if budget is not None and call_count >= int(budget): - - return {"allowed": False, "reason": "budget_exceeded", "tool_id": tool_id, - - "message": policy.get("fallback_message") - - or "该智能体已达到工具调用上限，请开启新对话或调整智能体设置。"} - - per_round = policy.get("per_round_cap") - - if per_round is not None and round_count >= int(per_round): - - return {"allowed": False, "reason": "per_round_cap", "tool_id": tool_id, - - "message": "本轮工具调用已达上限"} - - return {"allowed": True, "tool_id": tool_id} - +# Auto-extracted from chat.py during Phase 1 refactor (Strangler Fig). + +# Source of truth: scripts/phase1_extract.py. Do not hand-merge back. + + + +from fastapi import APIRouter, Depends, HTTPException, Query, Response + +from sqlalchemy.orm import Session + +from sqlalchemy import and_, desc, asc, func + +from pydantic import BaseModel, Field + +from typing import Optional + +import uuid + +from datetime import datetime, timezone, timedelta + +import re + +import json + +import asyncio + +import math + +import httpx + +import os + +import logging + +from ..database import get_db, SessionLocal + +from app.core.observability import span, get_trace_id + +from ..models import ( + + + + + + + + User, Conversation, Message, AIModel, ModelProvider, TokenUsageLog, + + + + + + + + KnowledgeBase, KnowledgeDocument, KnowledgeDocumentChunk, EvolutionSkill, + + + + + + + + EvolutionTrajectory, MarketApp, AppPurchase, AppInstallation, + + + + + + + + MCPServer, + + + + + + + + gen_uuid + + + + + + + +) + +from ..auth import get_current_user + +from ..platform_knowledge import get_platform_knowledge + +from ..mcp_client import list_mcp_tools, call_mcp_tool + +from ..middleware.log_service import LogService + +from ..capability_registry import resolve_auto_model + +from app.api.chat_agents import (_call_external_agent) + +from app.api.chat_context import (_build_system_prompt, _get_conversation_context, _ingest_answer_to_dify, _retrieve_kb_context, _build_chat_history, _maybe_compress_history) + +from app.api.chat_mcp import (_list_enabled_mcp_servers, _resolve_active_mcp_servers) + +from app.api.chat_media import (_generate_media_task, detect_ambiguous_creative_intent, detect_media_intent) + +from app.api.chat_skill import (_discover_installed_skills, _execute_skill_tool, _resolve_market_agents, _resolve_market_skills) + +from app.api.chat_utils import (SendMessageRequest, _proxy_for_url, cst_now, logger) + + + +async def _send_message_core( + + + + + + + + conv_id: str, + + + + + + + + req: SendMessageRequest, + + + + + + + + db: Session = Depends(get_db), + + + + + + + + current_user: User = Depends(get_current_user), + + + + + + + +): + + + + + + + + conv = db.query(Conversation).filter( + + + + + + + + and_(Conversation.id == conv_id, Conversation.user_id == current_user.id) + + + + + + + + ).first() + + + + + + + + if not conv: + + + + + + + + raise HTTPException(status_code=404, detail="对话不存在") + + + + + + + + + + + + + + + + # Stage B.2: 多轮上下文保持 — 如果当前请求没有skill/agent/kb, + + + + + + + + # 从对话历史中继承上一次的选择 + + + + + + + + effective_skill_ids = req.skill_ids or [] + + + + + + + + effective_agent_ids = req.agent_ids or [] + + + + + + + + effective_kb_ids = req.knowledge_base_ids or [] + + + + + + + + effective_mcp_ids = req.mcp_ids or [] + + + + + + + + effective_use_tools = req.use_tools or False + + + + + + + + + + + + + + + + if not effective_skill_ids and not effective_agent_ids and not effective_kb_ids and not effective_mcp_ids: + + + + + + + + # 从对话上下文继承 + + + + + + + + prev_context = _get_conversation_context(db, conv_id) + + + + + + + + if prev_context: + + + + + + + + effective_skill_ids = prev_context.get("skill_ids", []) + + + + + + + + effective_agent_ids = prev_context.get("agent_ids", []) + + + + + + + + effective_kb_ids = prev_context.get("knowledge_base_ids", []) + + + + + + + + effective_mcp_ids = prev_context.get("mcp_ids", []) + + + + + + + + if effective_skill_ids or effective_agent_ids or effective_mcp_ids: + + + + + + + + effective_use_tools = True + + + + + + + + + + + + + + + + # 附件内容拼接到用户消息中(文本类文件附带内容, 供模型参考) + + + + + + + + attachment_block = "" + + + + + + + + if req.attachments: + + + + + + + + parts = [] + + + + + + + + for att in req.attachments: + + + + + + + + name = att.get("name", "附件") if isinstance(att, dict) else str(att) + + + + + + + + ftype = att.get("file_type", "") if isinstance(att, dict) else "" + + + + + + + + content = att.get("content", "") if isinstance(att, dict) else "" + + + + + + + + if content: + + + + + + + + parts.append(f"【附件: {name}】\n{content}") + + + + + + + + else: + + + + + + + + parts.append(f"【附件(二进制, 名称: {name}, 类型: {ftype})】") + + + + + + + + if parts: + + + + + + + + attachment_block = "\n\n" + "\n\n".join(parts) + + + + + + + + + + + + + + + + # 保存用户消息(含skill/agent/kb/mcp选择用于上下文保持) + + + + + + + + user_msg = Message( + + + + + + + + id=gen_uuid(), + + + + + + + + conversation_id=conv_id, + + + + + + + + role="user", + + + + + + + + content=req.content + attachment_block, + + + + + + + + token_used=len(req.content) // 2, + + + + + + + + model_id=req.model_id or conv.model_id, + + + + + + + + skill_ids=req.skill_ids if req.skill_ids else None, + + + + + + + + agent_ids=req.agent_ids if req.agent_ids else None, + + + + + + + + knowledge_ids=req.knowledge_base_ids if req.knowledge_base_ids else None, + + + + + + + + mcp_ids=req.mcp_ids if req.mcp_ids else None, + + + + + + + + attachments=req.attachments if req.attachments else None, + + + + + + + + created_at=cst_now(), + + + + + + + + ) + + + + + + + + db.add(user_msg) + + + + + + + + + + + + + + + + # ===== 模糊创作意图拦截 ===== + + + + + + + + # 用户使用创作类动词(制作/创作/做/生成…)但未明确产出形式时，先与用户确认方向， + + + + + + + + # 避免直接调用大模型被内容安全策略拒答(如返回 "high risk")。 + + + + + + + + clarify = detect_ambiguous_creative_intent(req.content) + + + + + + + + if clarify: + + + + + + + + clarify_content = "<>" + json.dumps( + + + + + + + + {"prompt": req.content, "options": clarify["options"]}, ensure_ascii=False + + + + + + + + ) + "<>" + + + + + + + + ai_msg = Message( + + + + + + + + id=gen_uuid(), + + + + + + + + conversation_id=conv_id, + + + + + + + + role="assistant", + + + + + + + + content=clarify_content, + + + + + + + + token_used=0, + + + + + + + + model_id=req.model_id or conv.model_id, + + + + + + + + created_at=cst_now(), + + + + + + + + ) + + + + + + + + db.add(ai_msg) + + + + + + + + db.commit() + + + + + + + + db.refresh(user_msg) + + + + + + + + db.refresh(ai_msg) + + + + + + + + return { + + + + + + + + "code": 0, + + + + + + + + "data": { + + + + + + + + "user_message": { + + + + + + + + "id": user_msg.id, "role": user_msg.role, "content": user_msg.content, + + + + + + + + "token_used": user_msg.token_used, "created_at": str(user_msg.created_at), + + + + + + + + }, + + + + + + + + "assistant_message": { + + + + + + + + "id": ai_msg.id, "role": ai_msg.role, "content": clarify_content, + + + + + + + + "created_at": str(ai_msg.created_at), "clarify": clarify, + + + + + + + + }, + + + + + + + + }, + + + + + + + + "message": "请选择创作方向", + + + + + + + + } + + + + + + + + + + + + + + + + # ===== 核心聊天引擎 ===== + + + + + + + + user_id = current_user.id + + + + + + + + model_id = req.model_id or conv.model_id + + + + + + + + prompt_tokens = 0 + + + + + + + + completion_tokens = 0 + + + + + + + + total_tokens = 0 + + + + + + + + ai_content = "" + + + + + + + + + + + + + + + + ai_model = None + + + + + + + + skills = [] # 初始化避免UnboundLocalError + process_steps = [] # 过程步骤(思考/技能/MCP执行)，用于前端收拢式展示 + + + + + + + + + agents = [] # 初始化避免UnboundLocalError + + + + + + + + media_generating = False # 提前初始化：外部Agent分支与末尾均会引用 + + + + + + + + external_handled = False # 是否已由外部Agent(Dify/n8n/Coze)远程处理 + + + + + + + + + + + + + + + + # ===== 外部 Agent 远程调用分支 (Dify / n8n / Coze) ===== + + + + + + + + # 若本轮对话关联了带有效 agent_endpoint 的外部 Agent，则直接远程调用其平台， + + + + + + + + # 跳过本地 LLM「扮演」逻辑。effective_agent_ids 已在前面从请求/上下文继承。 + + + + + + + + external_agents = _resolve_market_agents(db, effective_agent_ids, user_id) + + + + + + + + external_agent = next((a for a in external_agents if a.get("agent_endpoint")), None) + + + + # ===== 3.3 多 Agent 编排(图引擎)快路 ===== + + _orchestrator = next((a for a in external_agents if a.get("orchestration")), None) + + if _orchestrator: + + try: + + from app.core.orchestrator import run_graph + + _answer, _trace = await run_graph( + + graph=_orchestrator["orchestration"], + + query=req.content + attachment_block, + + db=db, + + user_id=user_id, + + conv_id=conv_id, + + trace_id=get_trace_id(), + + ) + + ai_content = _answer or "编排未产生内容。" + + logger.info( + + f"[orchestrator] 图执行完成 trace_id={get_trace_id()} nodes={len(_trace.get('nodes', []))}" + + ) + + except Exception as e: + + logger.warning(f"[orchestrator] 图执行失败: {e}") + + ai_content = f"⚠️ 多 Agent 编排执行失败：{str(e)[:200]}" + + external_handled = True + + external_agent = None # 阻止下方外部 Agent 分支重复执行 + + + + + + + + if external_agent: + + + + + + + + agents = external_agents + + + + + + + + # 组装最近历史(供 n8n 等无平台会话态的服务维持多轮上下文) + + + + + + + + _history = _build_chat_history(db, conv_id, max_tokens=4000, exclude_latest_user=True) + + + + + + + + try: + + + + + + + + answer, new_ext_id = await _call_external_agent( + + + + + + + + db, external_agent, + + + + + + + + query=req.content + attachment_block, + + + + + + + + user_id=user_id, + + + + + + + + ext_conversation_id=conv.ext_agent_conversation_id, + + + + + + + + history=_history, + + + + + + + + ) + + + + + + + + ai_content = answer or "外部 Agent 未返回内容。" + + + + + + + + if new_ext_id: + + + + + + + + conv.ext_agent_conversation_id = new_ext_id + + + + + + + + except Exception as e: + + + + + + + + logger.warning( + + + + + + + + f"[external_agent] 远程调用失败 platform={external_agent.get('platform')} " + + + + + + + + f"endpoint={external_agent.get('agent_endpoint')}: {e}" + + + + + + + + ) + + + + + + + + ai_content = f"⚠️ 外部 Agent 调用失败：{str(e)[:200]}" + + + + + + + + external_handled = True + + + + + + + + + + + + + + + + # 解析模型：支持 Auto 模式（model_id='auto' 或未指定时，按能力自动匹配平台默认模型） + + + + + + + + if not external_handled: + + + + + + + + if model_id and model_id != "auto": + + + + + + + + ai_model = db.query(AIModel).filter(AIModel.id == model_id).first() + + + + + + + + if not ai_model: + + + + + + + + # Auto / 未指定 → 按用户输入意图匹配对应能力的默认模型，否则回退文本对话 + + + + + + + + intent = detect_media_intent(req.content) + + + + + + + + resolved = resolve_auto_model(intent, db) if intent else None + + + + + + + + ai_model = resolved or resolve_auto_model("text", db) + + + + + + + + + + + + + + + + if ai_model: + + + + + + + + # ====== 图像/视频生成模型分支 ====== + + + + + + + + caps = [c.lower() for c in (ai_model.capabilities or [])] if ai_model else [] + + + + + + + + media_generating = False + + + + + + + + if ai_model and ("image" in caps or "video" in caps): + + + + + + + + # 异步生成：立即返回占位内容，后台任务轮询 AGNES 并回写， + + + + + + + + # 避免最长 5 分钟的视频轮询阻塞 HTTP 请求导致客户端超时断开。 + + + + + + + + media_generating = True + + + + + + + + if "video" in caps: + + + + + + + + ai_content = "⏳ 视频生成中…（通常 1~3 分钟，完成后自动刷新）" + + + + + + + + else: + + + + + + + + ai_content = "⏳ 图像生成中…（完成后自动刷新）" + + + + + + + + elif ai_model and ai_model.endpoint: + + + + + + + + provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first() + + + + + + + + api_key = ai_model.api_key or (provider.api_key if provider else None) + + + + + + + + api_url = ai_model.endpoint + + + + + + + + + + + + + + + + if not api_url and provider and provider.api_base_url: + + + + + + + + api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions" + + + + + + + + + + + + + + + + if api_key and api_url: + + + + + + + + try: + + + + + + + + # Stage A+B: 解析Skills和Agents + + + + + + + + skills = _resolve_market_skills(db, effective_skill_ids, user_id) + + + + + + + + agents = _resolve_market_agents(db, effective_agent_ids, user_id) + + + + + + + + mcps = _resolve_active_mcp_servers(db, effective_mcp_ids) + + + + + + + + # MCP 工具调用链路：拉取各 MCP 工具并转为 function definitions + + + + mcp_servers_conn = mcps + + + + mcp_function_definitions = [] + + + + mcp_tool_map = {} + + + + mcp_idx = 0 + + + + for _srv in mcp_servers_conn: + + + + try: + + + + async with span("mcp.list_tools", server=_srv.get("name") if isinstance(_srv, dict) else None): + + _tools = await list_mcp_tools(_srv) + + + + except Exception as _e: + + + + logger.warning(f"MCP 拉取工具失败 [{_srv.get('name')}]: {_e}") + + + + _tools = [] + + + + if _tools: + + + + try: + + + + _upd = db.query(MCPServer).filter(MCPServer.id == _srv["id"]).first() + + + + if _upd: + + + + _upd.tools_count = len(_tools) + + + + db.commit() + + + + except Exception: + + + + pass + + + + for _t in _tools: + + + + _fname = f"mcp_tool_{mcp_idx}" + + + + mcp_tool_map[_fname] = { + + + + "server_id": _srv["id"], + + + + "name": _t.get("name"), + + + + "server_name": _srv["name"], + + + + } + + + + _schema = _t.get("inputSchema") or {"type": "object", "properties": {}} + + + + if not isinstance(_schema, dict) or _schema.get("type") != "object": + + + + _schema = {"type": "object", "properties": {}} + + + + mcp_function_definitions.append({ + + + + "name": _fname, + + + + "description": f"[MCP·{_srv['name']}] {_t.get('description', '')}", + + + + "parameters": _schema, + + + + }) + + + + mcp_idx += 1 + + + + if mcp_function_definitions: + + + + effective_use_tools = True + + + + + + + + + + + + + + + + # Issue 8: 默认技能自动发现（仅当用户未显式选择技能时启用） + + + + + + + + # 每轮先寻找本账号已安装且平台已部署的技能，多个适用时由模型自行判断选 1 个最优。 + + + + + + + + used_auto_skills = False + + + + + + + + if not effective_skill_ids: + + + + + + + + try: + + + + + + + + discovered = _discover_installed_skills(db, user_id) + + + + + + + + except Exception as e: + + + + + + + + logger.warning(f"自动发现已安装技能失败: {e}") + + + + + + + + discovered = [] + + + + + + + + if discovered: + + + + + + + + skills = discovered + + + + + + + + used_auto_skills = True + + + + + + + + effective_use_tools = True + + + + + + + + + + + + + + + + # Stage B.1: Agent可能自带skill_ids和kb_ids + + + + + + + + agent_extra_skill_ids = [] + + + + + + + + agent_extra_kb_ids = [] + + + + + + + + agent_model_override = None + + + + + + + + for agent in agents: + + + + + + + + if agent.get("skill_ids"): + + + + + + + + agent_extra_skill_ids.extend(agent["skill_ids"]) + + + + + + + + if agent.get("knowledge_base_ids"): + + + + + + + + agent_extra_kb_ids.extend(agent["knowledge_base_ids"]) + + + + + + + + if agent.get("model_id"): + + + + + + + + agent_model_override = agent["model_id"] + + + + + + + + + + + + + + + + # 合并Agent自带的skills和KBs + + + + + + + + all_skill_ids = list(set(effective_skill_ids + agent_extra_skill_ids)) + + + + + + + + all_kb_ids = list(set(effective_kb_ids + agent_extra_kb_ids)) + + + + + + + + + + + + + + + + if agent_extra_skill_ids or agent_extra_kb_ids: + + + + + + + + # 重新解析合并后的skills + + + + + + + + skills = _resolve_market_skills(db, all_skill_ids, user_id) + + + + + + + + effective_use_tools = True + + + + + + + + + + + + + + + + # Stage C: 增强版KB检索 + + + + + + + + kb_context = await _retrieve_kb_context(db, all_kb_ids, req.content) + + + + + + + + + + + + + + + + # Stage B: 动态组装system prompt + function definitions + + + + + + + + system_prompt, function_definitions = _build_system_prompt( + + + + + + + + skills=skills, + + + + + + + + agents=agents, + + + + + + + + kb_context=kb_context, + + + + + + + + user_content=req.content, + + + + + + + + mcp=mcps, + + + + mcp_functions=mcp_function_definitions, + + + + auto_discovered=used_auto_skills, + + all_mcp=_list_enabled_mcp_servers(db), + + db=db, + + ) + + + + + + + + + + + + + + + + # 构建消息列表 + + + + + + + + # 实际调用追踪（Issue：反馈真实使用的技能/MCP，而非注入的全部） + + called_skill_names = [] + + called_mcp_names = [] + + + + messages = [{"role": "system", "content": system_prompt}] + + + + + + + + + + + + + + + + # 分层上下文引擎：预算感知历史 + 压缩摘要(若有) + + messages.extend(_build_chat_history(db, conv_id, max_tokens=6000, exclude_latest_user=True)) + + + + + + + + + + + + + + + + # Issue 1: 必须把附件内容(attachment_block)一并发给模型， + + + + + + + + # 否则用户在对话窗上传文件后模型“看不到”文件内容。 + + + + + + + + # Issue 2 修复：用户输入含 URL 且要求读取/加入/安装时，服务端抓取真实页面内容注入上下文 + try: + from app.api.url_reader import fetch_url_context + _url_ctx = await fetch_url_context(req.content) + except Exception as _e: + import logging as _logging + _logging.getLogger(__name__).warning("URL 上下文抓取失败(已跳过): %s", _e) + _url_ctx = None + if _url_ctx and _url_ctx.get("text"): + messages.append({ + "role": "system", + "content": ( + "【用户随消息提供的参考链接的真实内容】\n" + f"链接地址: {_url_ctx.get('url')}\n" + "下面是该链接页面的真实内容，请严格基于它来回答用户的需求" + "（例如按其『Quick Start / 加入步骤』逐步引导用户完成安装或加入），" + "不要凭空编造步骤：\n\n" + _url_ctx["text"] + ), + }) + process_steps.append({ + "type": "skill", + "name": "网页/仓库读取", + "result": f"已读取 {_url_ctx.get('url')}（{len(_url_ctx['text'])} 字符）", + }) + messages.append({"role": "user", "content": req.content + attachment_block}) + + + + + + + + + + + + + + + + # Stage B.3: Function Calling循环(最多5轮) + + + + + + + + max_tool_rounds = 5 + + + + + + + + # ===== 3.1 tool_policy 沙箱：聚合策略 + 初始化预算计数器 ===== + + _agent_tool_policy = _resolve_agent_tool_policy(agents) + + _tool_call_count = 0 + + for round_num in range(max_tool_rounds): + + _round_tool_count = 0 + + + + + + + + request_body = { + + + + + + + + "model": ai_model.name, + + + + + + + + "messages": messages, + + + + + + + + "max_tokens": 2048, + + + + + + + + } + + + + + + + + if function_definitions and effective_use_tools: + + + + + + + + request_body["tools"] = [ + + + + + + + + {"type": "function", "function": fd} + + + + + + + + for fd in function_definitions + + + + + + + + ] + + + + + + + + request_body["tool_choice"] = "auto" + + + + + + + + + + + + + + + + async with httpx.AsyncClient( + + + + + + + + timeout=ai_model.timeout or 120, + + + + + + + + proxy=_proxy_for_url(api_url), + + + + + + + + trust_env=False, + + + + + + + + ) as client: + + + + + + + + async with span("llm.completion", model=ai_model.id if ai_model else None): + + resp = await client.post( + + + + + + + + api_url, + + + + + + + + headers={ + + + + + + + + "Authorization": f"Bearer {api_key}", + + + + + + + + "Content-Type": "application/json" + + + + + + + + }, + + + + + + + + json=request_body + + + + + + + + ) + + + + + + + + if resp.status_code != 200: + + + + + + + + if round_num == 0: + + + + + + + + ai_content = f"AI模型调用失败 (HTTP {resp.status_code}), 请检查模型配置。" + + + + + + + + break + + + + + + + + + + + + + + + + data = resp.json() + + + + + + + + choice = data.get("choices", [{}])[0] + + + + + + + + message = choice.get("message", {}) + + + + + + + + + + + + + + + + # 记录token使用量 + + + + + + + + usage = data.get("usage", {}) + + + + + + + + prompt_tokens += usage.get("prompt_tokens", 0) + + + + + + + + completion_tokens += usage.get("completion_tokens", 0) + + + + + + + + total_tokens += usage.get("total_tokens", 0) + + + + + + + + + + + + + + + + # 检查是否有tool_calls + + + + + + + + tool_calls = message.get("tool_calls", []) + # 过程步骤：捕获模型思考/推理(产生工具调用前的推理) + _think = (message.get("content") or "").strip() + if _think and tool_calls: + process_steps.append({ + "type": "thinking", + "text": _think, + }) + + + + + + + + + if tool_calls and effective_use_tools: + + + + + + + + # 将assistant消息(含tool_calls)加入上下文 + + + + + + + + messages.append({ + + + + + + + + "role": "assistant", + + + + + + + + "content": message.get("content") or "", + + + + + + + + "tool_calls": tool_calls, + + + + + + + + }) + + + + + + + + + + + + + + + + # Stage A.2: 真实Skill执行 + + + + + + + + for tc in tool_calls: + + + + + + + + func_name = tc.get("function", {}).get("name", "") + + # ===== 3.1 tool_policy 沙箱：逐工具校验（白名单 + 预算 + 每轮上限） ===== + + _mcp_info = mcp_tool_map.get(func_name) if func_name.startswith("mcp_tool_") else None + + _policy_check = _check_tool_policy(func_name, _mcp_info, skills, _agent_tool_policy, _tool_call_count, _round_tool_count) + + if not _policy_check["allowed"]: + + _reason = _policy_check["reason"] + + logger.warning( + + f"[tool_policy] blocked tool call: agent={_agent_tool_policy.get('agent_id') if _agent_tool_policy else None} " + + f"tool={func_name} reason={_reason} trace={get_trace_id()}" + + ) + + tool_result = f"[工具调用已被策略拦截] {_policy_check.get('message', _reason)}" + + messages.append({ + + "role": "tool", + + "tool_call_id": tc.get("id", f"call_{round_num}"), + + "content": tool_result, + + }) + + _tool_call_count += 1 + + _round_tool_count += 1 + + continue + + _tool_call_count += 1 + + _round_tool_count += 1 + + + + + + + + func_args_str = tc.get("function", {}).get("arguments", "{}") + + + + + + + + try: + + + + + + + + func_args = json.loads(func_args_str) + + + + + + + + except Exception: + + + + + + + + func_args = {"input": func_args_str} + + + + + + + + + + + + + + + + if func_name.startswith("mcp_tool_"): + + + + # MCP 工具调用：代理到对应 MCP 服务执行 tools/call + + + + _info = mcp_tool_map.get(func_name) + + + + tool_result = "[MCP 工具未找到或映射缺失]" + + + + if _info: + + + + _srv = next( + + + + (s for s in mcp_servers_conn if s["id"] == _info["server_id"]), + + + + None, + + + + ) + + + + if _srv: + + + + async with span(f"mcp.call:{_info.get('name') if isinstance(_info, dict) else None}"): + + tool_result = await call_mcp_tool( + + + + _srv, _info["name"], func_args + + + + ) + + + + called_mcp_names.append(_srv.get("name", "")) + process_steps.append({ + "type": "mcp", + "name": _info.get("name") if isinstance(_info, dict) else func_name, + "server": _srv.get("name", ""), + "args": func_args, + "result": (tool_result or "")[:800], + }) + + + + + else: + + + + async with span(f"skill.execute:{func_name}"): + + tool_result = _execute_skill_tool( + + + + db, func_name, func_args, skills, req.content + + + + ) + + + + _sk = next((s for s in skills if s.get("func_name") == func_name), None) + + + + if _sk: + + + + called_skill_names.append(_sk.get("name", "")) + process_steps.append({ + "type": "skill", + "name": (_sk.get("name") if _sk else func_name), + "args": func_args, + "result": (tool_result or "")[:800], + }) + + + + + + + + + messages.append({ + + + + + + + + "role": "tool", + + + + + + + + "tool_call_id": tc.get("id", f"call_{round_num}"), + + + + + + + + "content": tool_result, + + + + + + + + }) + + + + + + + + + + + + + + + + # 继续下一轮让模型处理tool结果 + + + + + + + + continue + + + + + + + + else: + + + + + + + + # 普通回复 + + + + + + + + ai_content = message.get("content", "") + + + + + + + + if not ai_content and round_num > 0: + + + + + + + + ai_content = "AI模型已处理工具调用结果, 但返回了空响应。" + + + + + + + + break + + + + + + + + + + + + + + + + if not ai_content: + + + + + + + + ai_content = "AI模型返回了空响应, 请稍后重试。" + + + + + + + + + + + + + + + + except Exception as e: + + + + + + + + logger.warning(f"AI API调用异常: {e}") + + + + + + + + ai_content = "AI服务暂时不可用, 请稍后重试或检查模型API配置。" + + + + + + + + else: + + + + + + + + ai_content = "该模型缺少API密钥或端点配置, 请在后台管理-大模型库中配置API Key和Endpoint。" + + + + + + + + else: + + + + + + + + ai_content = "请选择一个已配置的AI模型进行对话。" + + + + + + + + elif not external_handled: + + + + + + + + ai_content = "请先在后台管理-大模型库配置默认对话模型（或聊天中选择模型），Auto 模式下将自动使用平台默认模型。" + + + + + + + + + + + + + + + + ai_token_count = len(ai_content) // 2 + + + + + + + + ai_msg = Message( + + + + + + + + id=gen_uuid(), + + + + + + + + conversation_id=conv_id, + + + + + + + + role="assistant", + + + + + + + + content=ai_content, + + + + + + + + token_used=ai_token_count, + + + + + + + + model_id=ai_model.id if ai_model else (req.model_id or conv.model_id), + + + + + + + + skill_ids=req.skill_ids if req.skill_ids else None, + + + + + + + + agent_ids=req.agent_ids if req.agent_ids else None, + + + + + + + + knowledge_ids=req.knowledge_base_ids if req.knowledge_base_ids else None, + + + + + + + + created_at=cst_now(), + + + + + + + + ) + + + + + + + + db.add(ai_msg) + + + + + + + + + + + + + + + + # 后台异步：将实质性文本回答入库到 Dify 知识库（量化后可供 RAG 复用） + + + + + + + + _ingest_text = (ai_content or "").strip() + + + + + + + + if len(_ingest_text) > 120 and not ( + + + + + + + + _ingest_text.startswith("[MEDIA") + + + + + + + + or _ingest_text.startswith("data:") + + + + + + + + or _ingest_text.startswith("http") + + + + + + + + or " 反馈配对的技能名 + + + + + + + + try: + + + + + + + + _sel = _resolve_market_skills(db, effective_skill_ids, user_id) + + + + + + + + used_skills = [s["name"] for s in _sel] + + + + + + + + except Exception: + + + + + + + + used_skills = [] + + + + + + + + elif called_skill_names: + + + + + + + + # 未显式选技能 -> 反馈 AI 本轮实际调用的技能（可能为空） + + + + + + + + used_skills = list(dict.fromkeys(called_skill_names)) + + + + + + + + if effective_mcp_ids: + + + + + + + + # 用户显式选择了 MCP -> 反馈配对的 MCP 名 + + + + + + + + try: + + + + + + + + _sel_mcp = _resolve_active_mcp_servers(db, effective_mcp_ids) + + + + + + + + used_mcp = [m["name"] for m in _sel_mcp] + + + + + + + + except Exception: + + + + + + + + used_mcp = [] + + + + + + + + elif called_mcp_names: + + + + + + + + used_mcp = list(dict.fromkeys(called_mcp_names)) + + + + + + + + ai_msg.used_skills = used_skills + + + + + + + + ai_msg.used_mcp = used_mcp + + + + + + + + + + + + + + + + # 更新对话统计 + + + + + + + + # 注意：knowledge_graph_count 仅应在用户实际点击「转知识图谱」时 +1， + # 不可在每次 AI 回复时无条件自增，否则未操作也会累积计数（见 #715）。 + + + + + + + + conv.updated_at = cst_now() + + + + + + + + + + + + + + + + # 记录Token使用（媒体生成走 _generate_media_task 异步补写真实消耗，此处跳过避免占位文本重复记账） + + + + + + + + if not media_generating: + + + + + + + + token_usage = TokenUsageLog( + + + + + + + + id=gen_uuid(), + + + + + + + + user_id=current_user.id, + + + + + + + + model_id=ai_model.id if ai_model else (req.model_id or conv.model_id), + + + + + + + + model_name=ai_model.name if ai_model else None, + + + + + + + + conversation_id=conv_id, + + trace_id=get_trace_id(), + + + + + + + + + + prompt_tokens=prompt_tokens or len(req.content) // 2, + + + + + + + + completion_tokens=completion_tokens or ai_token_count, + + + + + + + + total_tokens=total_tokens or (len(req.content) // 2 + ai_token_count), + + + + + + + + created_at=cst_now(), + + + + + + + + ) + + + + + + + + db.add(token_usage) + + + + + + + + + + + + + + + + # 进化轨迹记录 + + + + + + + + try: + + + + + + + + existing = db.query(EvolutionTrajectory).filter( + + + + + + + + EvolutionTrajectory.conversation_id == conv_id + + + + + + + + ).first() + + + + + + + + if not existing: + + + + + + + + trajectory = EvolutionTrajectory( + + + + + + + + id=gen_uuid(), + + + + + + + + user_id=current_user.id, + + + + + + + + conversation_id=conv_id, + + + + + + + + status="discovered", + + + + + + + + token_used=ai_token_count, + + + + + + + + created_at=cst_now(), + + + + + + + + ) + + + + + + + + db.add(trajectory) + + + + + + + + db.flush() + + + + + + + + + + + + + + + + try: + + + + + + + + from app.services.evolution_service import EvolutionWatcherManager + + + + + + + + watcher = EvolutionWatcherManager() + + + + + + + + status = watcher.get_user_status(current_user.id, lambda: SessionLocal()) + + + + + + + + if not status.get("scheduler_running"): + + + + + + + + watcher.start_user_evolution(current_user.id, lambda: SessionLocal()) + + + + + + + + logger.info(f"[Evolution] 自动启动用户 {current_user.id} 的进化Pipeline") + + + + + + + + except Exception as evolution_err: + + + + + + + + logger.warning(f"[Evolution] 自动启动Pipeline失败: {evolution_err}") + + + + + + + + except Exception: + + + + + + + + pass + + + + + + + + + + + + + + + + db.commit() + + + + + + + + # 后台异步压缩历史(分层记忆)，不阻塞本次响应 + + asyncio.create_task(_maybe_compress_history(conv_id)) + + + + db.refresh(user_msg) + + + + + + + + db.refresh(ai_msg) + + + + + + + + + + + + + + + + # 媒体生成改为后台异步：立即返回占位，后台任务完成后回写消息内容 + + + + + + + + if media_generating: + + + + + + + + try: + + + + + + + + asyncio.create_task( + + + + + + + + _generate_media_task(ai_msg.id, ai_model.id, req.content, current_user.id) + + + + + + + + ) + + + + + + + + except Exception as e: + + + + + + + + logger.warning(f"[_generate_media_task] 启动后台任务失败: {e}", exc_info=True) + # 启动失败兜底：不让占位消息永久停在「⏳ 生成中…」 + try: + ai_msg.content = "⚠️ 媒体生成启动失败，请稍后重试" + db.commit() + except Exception: + pass + + + + + + + + + + + + + + + + return { + + + + + + + + "code": 0, + + + + + + + + "data": { + + + + + + + + "user_message": { + + + + + + + + "id": user_msg.id, "role": user_msg.role, "content": user_msg.content, + + + + + + + + "token_used": user_msg.token_used, "created_at": str(user_msg.created_at), + + + + + + + + }, + + + + + + + + "assistant_message": { + + + + + + + + "id": ai_msg.id, "role": ai_msg.role, "content": ai_msg.content, + + + + + + + + "token_used": ai_msg.token_used, "created_at": str(ai_msg.created_at), + + + + + + + + "used_skills": used_skills, + + "used_mcp": used_mcp, + + + + + + + + "generating": media_generating, + "process_steps": process_steps, + + + + + + + + + }, + + + + + + + + }, + + + + + + + + "message": "发送成功", + + + + + + + + } + + + + + +# ====================================================================== + +# 3.1 tool_policy 沙箱辅助函数 + +# ====================================================================== + +def _resolve_agent_tool_policy(agents): + + """从已解析的 agents 聚合工具策略。多 agent 时取首个带有非空策略者。""" + + if not agents: + + return None + + for ag in agents: + + tp = ag.get("tool_policy") + + if tp and isinstance(tp, dict): + + tp = dict(tp) + + tp["agent_id"] = ag.get("id") + + return tp + + return None + + + + + +def _check_tool_policy(func_name, mcp_info, skills, policy, call_count, round_count): + + """校验单个工具调用是否被策略允许。 + + + + 返回 {"allowed": bool, "reason": str, "tool_id": str|None, "message": str} + + policy 为 None/空 -> 允许（向后兼容）。 + + """ + + if not policy: + + return {"allowed": True, "tool_id": None} + + allowed = policy.get("allowed_tools") or {} + + skill_ids = allowed.get("skill_ids") or [] + + mcp_ids = allowed.get("mcp_ids") or [] + + is_mcp = func_name.startswith("mcp_tool_") + + tool_id = None + + if is_mcp: + + if mcp_info and isinstance(mcp_info, dict): + + tool_id = mcp_info.get("server_id") + + if mcp_ids and tool_id not in mcp_ids: + + return {"allowed": False, "reason": "mcp_not_in_allowlist", "tool_id": tool_id, + + "message": "MCP 服务不在该智能体的允许列表中"} + + else: + + if skills: + + _s = next((s for s in skills if s.get("func_name") == func_name), None) + + if _s: + + tool_id = _s.get("id") + + if skill_ids and tool_id not in skill_ids: + + return {"allowed": False, "reason": "skill_not_in_allowlist", "tool_id": tool_id, + + "message": "技能不在该智能体的允许列表中"} + + budget = policy.get("max_calls_per_conversation") + + if budget is not None and call_count >= int(budget): + + return {"allowed": False, "reason": "budget_exceeded", "tool_id": tool_id, + + "message": policy.get("fallback_message") + + or "该智能体已达到工具调用上限，请开启新对话或调整智能体设置。"} + + per_round = policy.get("per_round_cap") + + if per_round is not None and round_count >= int(per_round): + + return {"allowed": False, "reason": "per_round_cap", "tool_id": tool_id, + + "message": "本轮工具调用已达上限"} + + return {"allowed": True, "tool_id": tool_id} + diff --git a/backend/python/app/api/chat_utils.py b/backend/python/app/api/chat_utils.py index 93dea39..d921713 100644 --- a/backend/python/app/api/chat_utils.py +++ b/backend/python/app/api/chat_utils.py @@ -1,387 +1,387 @@ -# Auto-extracted from chat.py during Phase 1 refactor (Strangler Fig). -# Source of truth: scripts/phase1_extract.py. Do not hand-merge back. - -from fastapi import APIRouter, Depends, HTTPException, Query, Response -from sqlalchemy.orm import Session -from sqlalchemy import and_, desc, asc, func -from pydantic import BaseModel, Field -from typing import Optional -import uuid -from datetime import datetime, timezone, timedelta -import re -import json -import asyncio -import math -import httpx -import os -import logging -from ..database import get_db, SessionLocal -from ..models import ( - - - - User, Conversation, Message, AIModel, ModelProvider, TokenUsageLog, - - - - KnowledgeBase, KnowledgeDocument, KnowledgeDocumentChunk, EvolutionSkill, - - - - EvolutionTrajectory, MarketApp, AppPurchase, AppInstallation, - - - - MCPServer, - - - - gen_uuid - - - -) -from ..auth import get_current_user -from ..platform_knowledge import get_platform_knowledge -from ..mcp_client import list_mcp_tools, call_mcp_tool -from ..middleware.log_service import LogService -from ..capability_registry import resolve_auto_model - -logger = logging.getLogger(__name__) - -CST_TZ = timezone(timedelta(hours=8)) - -def cst_now(): - - - - """Get current time in CST timezone""" - - - - return datetime.now(CST_TZ) - -def _html_escape(s: str) -> str: - - - - return (str(s).replace("&", "&").replace("<", "<").replace(">", ">") - - - - .replace('"', """).replace("'", "'")) - -def _outbound_proxy() -> str: - - - - """出网代理地址(兜底)。默认经 172.18.0.1:7890 智能路由代理； - - - - 运行时若未注入代理环境变量则回退到该地址。""" - - - - import os - - - - return os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or "http://172.18.0.1:7890" - -_HK_PROXY = "http://qyyfw:XHR521yhf@103.242.14.208:11495" - -def _proxy_for_url(url: str) -> str | None: - - - - """按目标域名选择是否走代理。 - - - - - - - - 2026-07-15 调整：AGNES(apihub.agnes-ai.com / platform-outputs.agnes-ai.space) - - - - 经用户实测走香港代理 103.242.14.208:11495 延迟更低、更稳定，故改为走 HK 代理 - - - - (带鉴权)，不再直连，也不经智能路由代理(172.18.0.1:7890 对 AGNES 返 502)。 - - - - 其余国内/平台域名直连，真正的境外域名走 _outbound_proxy()。 - - - - - - - - 注意：httpx 默认 trust_env=True 会忽略 proxy 参数而改用环境变量 HTTP_PROXY， - - - - 故调用方必须同时传 trust_env=False 才能真正走此处指定的代理。 - - - - """ - - - - import os - - - - from urllib.parse import urlparse - - - - host = (urlparse(url).hostname or "").lower() - - - - # 文生图 / 视频生成走直连：智能路由代理(172.18.0.1:7890)与 HK 代理对这类长耗时、 - # 大响应体的生成连接会偶发重置(Server disconnected)，而各图像/视频供应商均为公网 - # API，直连更稳定(已实测 gpt-image-2 / agnes 直连均返回 200)。故媒体端点统一直连。 - _lower = (url or "").lower() - if "/images/" in _lower or "/videos/" in _lower or "/v1/images" in _lower or "/v1/videos" in _lower: - return None - - - - if "agnes-ai.com" in host or "agnes-ai.space" in host: - - - - return _HK_PROXY - - - - # 平台自有域名(含自建 Dify/n8n 等)必须直连，不经出网代理。 - - - - if "qihuyun.com" in host: - - - - return None - - - - # 尊重 NO_PROXY：国内域名(硅基流动/DeepSeek/小米MoE 等)直连更稳定， - - - - # 避免经易波动的智能路由代理 172.18.0.1:7890(对国内域名偶发连接重置)。 - - - - _no_proxy = [h.strip().lower() for h in os.environ.get("NO_PROXY", "").split(",") if h.strip()] - - - - for _np in _no_proxy: - - - - if host == _np or host.endswith("." + _np) or _np in host: - - - - return None - - - - return _outbound_proxy() - -def _extract_text_from_json(obj) -> str: - - - - """从 n8n / 通用 webhook 的任意 JSON 响应中尽力抽取文本回答。""" - - - - if obj is None: - - - - return "" - - - - if isinstance(obj, str): - - - - return obj - - - - if isinstance(obj, (int, float, bool)): - - - - return str(obj) - - - - if isinstance(obj, list): - - - - # 取第一个可抽取到文本的元素 - - - - for it in obj: - - - - t = _extract_text_from_json(it) - - - - if t: - - - - return t - - - - return "" - - - - if isinstance(obj, dict): - - - - # 常见回答字段优先级 - - - - for k in ("output", "text", "answer", "response", "reply", - - - - "message", "content", "result", "data"): - - - - if k in obj: - - - - t = _extract_text_from_json(obj[k]) - - - - if t: - - - - return t - - - - # 兜底：整体序列化 - - - - try: - - - - return json.dumps(obj, ensure_ascii=False) - - - - except Exception: - - - - return str(obj) - - - - return str(obj) - -class CreateConversationRequest(BaseModel): - - - - title: str = Field(default="新对话") - - - - model_id: Optional[str] = None - -class UpdateConversationRequest(BaseModel): - - - - title: Optional[str] = None - - - - status: Optional[str] = None # active, archived - - - - pinned: Optional[bool] = None - - - - agent_ids: Optional[list] = None - - - - skill_ids: Optional[list] = None - - - - knowledge_base_ids: Optional[list] = None - -class SendMessageRequest(BaseModel): - - - - content: str - - - - model_id: Optional[str] = None - - - - skill_ids: Optional[list] = None - - - - agent_ids: Optional[list] = None - - - - knowledge_base_ids: Optional[list] = None - - - - mcp_ids: Optional[list] = None - - - - attachments: Optional[list] = None - - - - use_tools: Optional[bool] = False +# Auto-extracted from chat.py during Phase 1 refactor (Strangler Fig). +# Source of truth: scripts/phase1_extract.py. Do not hand-merge back. + +from fastapi import APIRouter, Depends, HTTPException, Query, Response +from sqlalchemy.orm import Session +from sqlalchemy import and_, desc, asc, func +from pydantic import BaseModel, Field +from typing import Optional +import uuid +from datetime import datetime, timezone, timedelta +import re +import json +import asyncio +import math +import httpx +import os +import logging +from ..database import get_db, SessionLocal +from ..models import ( + + + + User, Conversation, Message, AIModel, ModelProvider, TokenUsageLog, + + + + KnowledgeBase, KnowledgeDocument, KnowledgeDocumentChunk, EvolutionSkill, + + + + EvolutionTrajectory, MarketApp, AppPurchase, AppInstallation, + + + + MCPServer, + + + + gen_uuid + + + +) +from ..auth import get_current_user +from ..platform_knowledge import get_platform_knowledge +from ..mcp_client import list_mcp_tools, call_mcp_tool +from ..middleware.log_service import LogService +from ..capability_registry import resolve_auto_model + +logger = logging.getLogger(__name__) + +CST_TZ = timezone(timedelta(hours=8)) + +def cst_now(): + + + + """Get current time in CST timezone""" + + + + return datetime.now(CST_TZ) + +def _html_escape(s: str) -> str: + + + + return (str(s).replace("&", "&").replace("<", "<").replace(">", ">") + + + + .replace('"', """).replace("'", "'")) + +def _outbound_proxy() -> str: + + + + """出网代理地址(兜底)。默认经 172.18.0.1:7890 智能路由代理； + + + + 运行时若未注入代理环境变量则回退到该地址。""" + + + + import os + + + + return os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or "http://172.18.0.1:7890" + +_HK_PROXY = "http://qyyfw:XHR521yhf@103.242.14.208:11495" + +def _proxy_for_url(url: str) -> str | None: + + + + """按目标域名选择是否走代理。 + + + + + + + + 2026-07-15 调整：AGNES(apihub.agnes-ai.com / platform-outputs.agnes-ai.space) + + + + 经用户实测走香港代理 103.242.14.208:11495 延迟更低、更稳定，故改为走 HK 代理 + + + + (带鉴权)，不再直连，也不经智能路由代理(172.18.0.1:7890 对 AGNES 返 502)。 + + + + 其余国内/平台域名直连，真正的境外域名走 _outbound_proxy()。 + + + + + + + + 注意：httpx 默认 trust_env=True 会忽略 proxy 参数而改用环境变量 HTTP_PROXY， + + + + 故调用方必须同时传 trust_env=False 才能真正走此处指定的代理。 + + + + """ + + + + import os + + + + from urllib.parse import urlparse + + + + host = (urlparse(url).hostname or "").lower() + + + + # 文生图 / 视频生成走直连：智能路由代理(172.18.0.1:7890)与 HK 代理对这类长耗时、 + # 大响应体的生成连接会偶发重置(Server disconnected)，而各图像/视频供应商均为公网 + # API，直连更稳定(已实测 gpt-image-2 / agnes 直连均返回 200)。故媒体端点统一直连。 + _lower = (url or "").lower() + if "/images/" in _lower or "/videos/" in _lower or "/v1/images" in _lower or "/v1/videos" in _lower: + return None + + + + if "agnes-ai.com" in host or "agnes-ai.space" in host: + + + + return _HK_PROXY + + + + # 平台自有域名(含自建 Dify/n8n 等)必须直连，不经出网代理。 + + + + if "qihuyun.com" in host: + + + + return None + + + + # 尊重 NO_PROXY：国内域名(硅基流动/DeepSeek/小米MoE 等)直连更稳定， + + + + # 避免经易波动的智能路由代理 172.18.0.1:7890(对国内域名偶发连接重置)。 + + + + _no_proxy = [h.strip().lower() for h in os.environ.get("NO_PROXY", "").split(",") if h.strip()] + + + + for _np in _no_proxy: + + + + if host == _np or host.endswith("." + _np) or _np in host: + + + + return None + + + + return _outbound_proxy() + +def _extract_text_from_json(obj) -> str: + + + + """从 n8n / 通用 webhook 的任意 JSON 响应中尽力抽取文本回答。""" + + + + if obj is None: + + + + return "" + + + + if isinstance(obj, str): + + + + return obj + + + + if isinstance(obj, (int, float, bool)): + + + + return str(obj) + + + + if isinstance(obj, list): + + + + # 取第一个可抽取到文本的元素 + + + + for it in obj: + + + + t = _extract_text_from_json(it) + + + + if t: + + + + return t + + + + return "" + + + + if isinstance(obj, dict): + + + + # 常见回答字段优先级 + + + + for k in ("output", "text", "answer", "response", "reply", + + + + "message", "content", "result", "data"): + + + + if k in obj: + + + + t = _extract_text_from_json(obj[k]) + + + + if t: + + + + return t + + + + # 兜底：整体序列化 + + + + try: + + + + return json.dumps(obj, ensure_ascii=False) + + + + except Exception: + + + + return str(obj) + + + + return str(obj) + +class CreateConversationRequest(BaseModel): + + + + title: str = Field(default="新对话") + + + + model_id: Optional[str] = None + +class UpdateConversationRequest(BaseModel): + + + + title: Optional[str] = None + + + + status: Optional[str] = None # active, archived + + + + pinned: Optional[bool] = None + + + + agent_ids: Optional[list] = None + + + + skill_ids: Optional[list] = None + + + + knowledge_base_ids: Optional[list] = None + +class SendMessageRequest(BaseModel): + + + + content: str + + + + model_id: Optional[str] = None + + + + skill_ids: Optional[list] = None + + + + agent_ids: Optional[list] = None + + + + knowledge_base_ids: Optional[list] = None + + + + mcp_ids: Optional[list] = None + + + + attachments: Optional[list] = None + + + + use_tools: Optional[bool] = False diff --git a/backend/python/app/api/market.py b/backend/python/app/api/market.py index 7299cb8..135646e 100644 --- a/backend/python/app/api/market.py +++ b/backend/python/app/api/market.py @@ -691,7 +691,12 @@ def list_apps( "capability": app.capability or None, "category": app.category or None, "tags": app.tags or None, - "installed": (app.id in installed_app_ids or is_user_owned) if user_id else False, + # 官方导入并发布的技能默认全局已安装（见 #716：无论前台/后台导入，默认全局安装） + "installed": ( + app.id in installed_app_ids + or is_user_owned + or (app.source == "official" and app.status == "published") + ), "version": app.version or "1.0.0", "security_scan": app.security_scan, "verified": bool(app.verified), @@ -756,7 +761,8 @@ def get_app( "tags": app.tags or None, "created_at": str(app.created_at), "purchased": purchased, - "installed": installed, + # 官方导入并发布的技能默认全局已安装（见 #716） + "installed": installed or (app.source == "official" and app.status == "published"), "version": app.version or "1.0.0", "security_scan": app.security_scan, "verified": bool(app.verified), diff --git a/backend/python/app/api/url_reader.py b/backend/python/app/api/url_reader.py new file mode 100644 index 0000000..7d3222d --- /dev/null +++ b/backend/python/app/api/url_reader.py @@ -0,0 +1,163 @@ +"""服务端 URL 读取器（Issue 2 修复） + +当用户输入包含 URL 且明确要求『读取 / 加入 / 安装 / 分析 / 总结』等意图时， +服务端抓取该链接的真实内容并作为参考上下文注入对话，使模型能基于页面真实信息 +作答（例如按 GitHub 仓库 README 的 Quick Start 引导用户完成安装/加入）， +而不是凭空编造。 + +SSRF 防护：仅允许 http/https；解析域名 IP 后阻断内网/回环/链路本地/云元数据地址； +限制跳转、响应体大小与超时。 +""" +import ipaddress +import logging +import re +import socket +from urllib.parse import urlparse + +import httpx + +logger = logging.getLogger(__name__) + +_URL_RE = re.compile(r'https?://[^\s"\)\\\]>]+', re.IGNORECASE) +# 触发意图：显式要求读取/加入/安装/分析/总结/打开/访问 等 +_INTENT_RE = re.compile( + r'(read|join|install|fetch|summar|analys|open|visit|access|browse|' + r'读取|加入|安装|打开|访问|浏览|总结|分析|抓取|看下这个|查看该|这个网页|这个页面|这个链接|help me join)', + re.IGNORECASE, +) + +# 常见私有/保留网段（阻断内网扫描与云元数据） +_BLOCKED_NETS = [ + ipaddress.ip_network('10.0.0.0/8'), + ipaddress.ip_network('172.16.0.0/12'), + ipaddress.ip_network('192.168.0.0/16'), + ipaddress.ip_network('127.0.0.0/8'), + ipaddress.ip_network('169.254.0.0/16'), + ipaddress.ip_network('::1/128'), + ipaddress.ip_network('fc00::/7'), + ipaddress.ip_network('fe80::/10'), + ipaddress.ip_network('0.0.0.0/8'), +] + +_MAX_CHARS = 16000 +_TIMEOUT = 30.0 + + +def _is_blocked_host(host: str) -> bool: + host = (host or '').strip().lower() + if not host: + return True + if re.fullmatch(r'(\d{1,3}\.){3}\d{1,3}', host): + try: + ip = ipaddress.ip_address(host) + except ValueError: + return True + return any(ip in net for net in _BLOCKED_NETS) + try: + infos = socket.getaddrinfo(host, None) + for info in infos: + ipstr = info[4][0] + try: + ip = ipaddress.ip_address(ipstr) + except ValueError: + continue + if any(ip in net for net in _BLOCKED_NETS): + return True + except Exception: + return True + return False + + +async def _make_client(): + try: + return httpx.AsyncClient(trust_env=True, follow_redirects=True, timeout=_TIMEOUT) + except Exception: + return httpx.AsyncClient(trust_env=False, follow_redirects=True, timeout=_TIMEOUT) + + +async def _github_text(url: str): + p = urlparse(url) + if p.netloc.lower() not in ('github.com', 'www.github.com'): + return None + parts = [x for x in p.path.split('/') if x] + if len(parts) < 2: + return None + owner, repo = parts[0], parts[1] + if len(parts) >= 4 and parts[2] == 'blob': + ref = parts[3] + fpath = '/'.join(parts[4:]) + raw = f'https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{fpath}' + return await _fetch_text(raw) + api = f'https://api.github.com/repos/{owner}/{repo}/readme' + try: + async with await _make_client() as c: + r = await c.get(api, headers={ + 'Accept': 'application/vnd.github+json', + 'User-Agent': 'Mozilla/5.0', + }) + if r.status_code == 200: + import base64 + d = r.json() + return base64.b64decode(d.get('content', '')).decode('utf-8', 'replace') + except Exception as e: + logger.warning('github readme fetch failed: %s', e) + return None + + +async def _fetch_text(url: str) -> str: + async with await _make_client() as c: + r = await c.get(url, headers={'User-Agent': 'Mozilla/5.0'}) + r.raise_for_status() + data = r.content[:2_000_000] + ctype = r.headers.get('content-type', '') + text = data.decode('utf-8', 'replace') + if 'html' in ctype: + return _html_to_text(text) + return text + + +def _html_to_text(html: str) -> str: + html = re.sub(r'(?is)<(script|style|head|noscript|svg|template)[^>]*>.*?', ' ', html) + html = re.sub(r'(?is)', ' ', html) + html = re.sub(r'(?is)<[^>]+>', ' ', html) + import html as _html + text = _html.unescape(html) + text = re.sub(r'[ \t]+', ' ', text) + text = re.sub(r'\n{3,}', '\n\n', text) + return text.strip() + + +async def fetch_url_context(text: str): + """若有 URL 且命中读取意图，返回 {'url','text'}；否则返回 None。""" + if not text: + return None + m = _URL_RE.search(text) + if not m: + return None + url = m.group(0).rstrip('.,);]') + if not _INTENT_RE.search(text): + return None + try: + p = urlparse(url) + if p.scheme not in ('http', 'https'): + return None + if _is_blocked_host(p.hostname): + logger.warning('blocked host for url fetch: %s', p.hostname) + return None + except Exception: + return None + + try: + if 'github.com' in (p.hostname or '').lower(): + body = await _github_text(url) + if not body: + body = await _fetch_text(url) + else: + body = await _fetch_text(url) + except Exception as e: + logger.warning('url fetch failed: %s', e) + return None + + if not body or not body.strip(): + return None + return {'url': url, 'text': body[:_MAX_CHARS]} diff --git a/backend/python/app/core/orchestrator.py b/backend/python/app/core/orchestrator.py index 658c400..b98cc1b 100644 --- a/backend/python/app/core/orchestrator.py +++ b/backend/python/app/core/orchestrator.py @@ -1,395 +1,395 @@ -"""app/core/orchestrator.py — Phase 3.3 多 Agent 编排图引擎。 - -设计目标： -- 以轻量 DAG 描述多 Agent 协作：节点(llm/local/external/reduce) + 有向边(数据流)。 -- 拓扑分层后按层并行执行(asyncio.gather)，互相独立的节点真正并发。 -- 失败隔离：每个节点异常不影响其它分支；per-node on_error=halt|continue。 -- 节点执行复用既有原语(_build_system_prompt / _execute_skill_tool / _call_external_agent / - 3.1 工具沙箱)，完全向后兼容——agent.orchestration 为 null 时 chat_service 走原逻辑。 -- 并发安全：每个节点用独立 SessionLocal 会话做 DB 读，避免共享请求会话跨协程竞态。 -""" -from __future__ import annotations - -import asyncio -import json -import logging -from typing import Any, Optional - -import httpx - -logger = logging.getLogger("aiflow.orchestrator") - -VALID_NODE_TYPES = ("llm", "local", "external", "reduce") - - -class OrchestrationError(ValueError): - """图结构非法或执行被 halt。""" - - -# --------------------------------------------------------------------------- -# 校验 + 拓扑 -# --------------------------------------------------------------------------- -def validate_graph(graph: Any) -> None: - if not isinstance(graph, dict): - raise OrchestrationError("orchestration 必须为对象") - nodes = graph.get("nodes") - edges = graph.get("edges") - if not isinstance(nodes, list) or not nodes: - raise OrchestrationError("nodes 必须为非空数组") - if not isinstance(edges, list): - raise OrchestrationError("edges 必须为数组") - - ids = set() - for i, n in enumerate(nodes): - if not isinstance(n, dict): - raise OrchestrationError(f"节点 #{i} 必须为对象") - nid = n.get("id") - if not nid or not isinstance(nid, str): - raise OrchestrationError(f"节点 #{i} 缺少字符串 id") - if nid in ids: - raise OrchestrationError(f"节点 id 重复: {nid}") - ids.add(nid) - ntype = n.get("type") - if ntype not in VALID_NODE_TYPES: - raise OrchestrationError( - f"节点 {nid} 类型非法: {ntype} (允许 {', '.join(VALID_NODE_TYPES)})" - ) - if ntype in ("local", "external") and not n.get("agent_id"): - raise OrchestrationError(f"节点 {nid} 类型 {ntype} 必须指定 agent_id") - - for i, e in enumerate(edges): - if not isinstance(e, dict): - raise OrchestrationError(f"边 #{i} 必须为对象") - f = e.get("from") - t = e.get("to") - if f not in ids: - raise OrchestrationError(f"边 #{i} 的 from={f} 指向不存在的节点") - if t not in ids: - raise OrchestrationError(f"边 #{i} 的 to={t} 指向不存在的节点") - if f == t: - raise OrchestrationError(f"边 #{i} 存在自环: {f}") - - -def topological_layers(graph: dict) -> list: - """Kahn 算法返回分层(每层内节点可并行)；含环则抛错。""" - nodes = graph["nodes"] - edges = graph.get("edges", []) - indeg = {n["id"]: 0 for n in nodes} - adj = {n["id"]: [] for n in nodes} - for e in edges: - f, t = e["from"], e["to"] - adj[f].append(t) - indeg[t] += 1 - order = {n["id"]: i for i, n in enumerate(nodes)} - layers: list = [] - remaining = set(indeg) - while remaining: - layer = sorted([nid for nid in remaining if indeg[nid] == 0], key=lambda x: order[x]) - if not layer: - cycle = sorted(remaining, key=lambda x: order[x]) - raise OrchestrationError(f"检测到环路，无法拓扑排序: {cycle}") - layers.append(layer) - for nid in layer: - remaining.discard(nid) - for nxt in adj[nid]: - indeg[nxt] -= 1 - return layers - - -def _node_by_id(graph: dict, nid: str) -> dict: - for n in graph["nodes"]: - if n["id"] == nid: - return n - return {} - - -def _predecessors(graph: dict, nid: str) -> list: - return [e["from"] for e in graph.get("edges", []) if e["to"] == nid] - - -def _successors(graph: dict, nid: str) -> list: - return [e["to"] for e in graph.get("edges", []) if e["from"] == nid] - - -def _find_output_node(graph: dict) -> str: - if graph.get("output_node"): - return graph["output_node"] - sinks = [n["id"] for n in graph["nodes"] if not _successors(graph, n["id"])] - if len(sinks) == 1: - return sinks[0] - for s in sinks: - if _node_by_id(graph, s).get("type") == "reduce": - return s - if sinks: - return sinks[0] - return graph["nodes"][-1]["id"] - - -def _assemble_node_input(query: str, upstream: str, is_entry: bool) -> str: - """入口节点直接吃用户原问题；下游节点综合原问题 + 上游输出。""" - if is_entry: - return query or "" - if upstream: - return f"原始用户问题:\n{query}\n\n上游节点输出:\n{upstream}" - return query or "" - - -# --------------------------------------------------------------------------- -# 模型解析 + LLM 调用原语 -# --------------------------------------------------------------------------- -async def _resolve_model_and_endpoint(model_id, db): - from app.models import AIModel, ModelProvider - from ..capability_registry import resolve_auto_model - - ai_model = None - if model_id and model_id != "auto": - ai_model = db.query(AIModel).filter(AIModel.id == model_id).first() - if not ai_model: - ai_model = resolve_auto_model("text", db) - if not ai_model: - raise OrchestrationError("无法解析到可用模型") - provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first() - api_key = ai_model.api_key or (provider.api_key if provider else None) - api_url = ai_model.endpoint - if not api_url and provider and provider.api_base_url: - api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions" - if not api_url or not api_key: - raise OrchestrationError(f"节点模型 {ai_model.name} 缺少 endpoint 或 api_key") - return ai_model, api_url, api_key - - -async def _llm_complete(system_prompt: str, user_content: str, model_id, db, user_id) -> str: - from app.api.chat_utils import _proxy_for_url - from app.core.observability import span - - ai_model, api_url, api_key = await _resolve_model_and_endpoint(model_id, db) - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_content}, - ] - request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048} - async with span("orchestrator.llm", model=ai_model.id): - async with httpx.AsyncClient( - timeout=ai_model.timeout or 120, - proxy=_proxy_for_url(api_url), - trust_env=False, - ) as client: - resp = await client.post( - api_url, - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - json=request_body, - ) - if resp.status_code != 200: - raise OrchestrationError(f"LLM 调用失败 HTTP {resp.status_code}: {resp.text[:200]}") - data = resp.json() - return data["choices"][0]["message"].get("content") or "" - - -# --------------------------------------------------------------------------- -# local 节点：复用单 Agent system prompt + 受限 tool round + 3.1 沙箱 -# --------------------------------------------------------------------------- -async def _run_local_agent_turn(agent: dict, node_input: str, db, user_id, conv_id) -> dict: - from app.api.chat_skill import _resolve_market_skills, _execute_skill_tool - from app.api.chat_context import _build_system_prompt, _retrieve_kb_context - from app.api.chat_service import _resolve_agent_tool_policy, _check_tool_policy - from app.api.chat_utils import _proxy_for_url - - skills = _resolve_market_skills(db, agent.get("skill_ids") or [], user_id) - kb_context = _retrieve_kb_context(db, agent.get("knowledge_base_ids") or [], node_input) - system_prompt, function_definitions = _build_system_prompt( - skills=skills, - agents=[agent], - kb_context=kb_context, - user_content=node_input, - auto_discovered=False, - ) - ai_model, api_url, api_key = await _resolve_model_and_endpoint(agent.get("model_id"), db) - tool_policy = _resolve_agent_tool_policy([agent]) - - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": node_input}, - ] - called_skills: list = [] - tool_call_count = 0 - max_tool_rounds = 5 - last_content = "" - async with httpx.AsyncClient( - timeout=ai_model.timeout or 120, - proxy=_proxy_for_url(api_url), - trust_env=False, - ) as client: - for round_num in range(max_tool_rounds): - request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048} - if function_definitions: - request_body["tools"] = [ - {"type": "function", "function": fd} for fd in function_definitions - ] - request_body["tool_choice"] = "auto" - resp = await client.post( - api_url, - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - json=request_body, - ) - if resp.status_code != 200: - raise OrchestrationError(f"local 节点 LLM 失败 HTTP {resp.status_code}: {resp.text[:200]}") - data = resp.json() - message = data["choices"][0]["message"] - tool_calls = message.get("tool_calls") or [] - content = message.get("content") or "" - last_content = content - if not tool_calls or not function_definitions: - return {"text": content, "used_tools": called_skills} - messages.append({"role": "assistant", "content": content, "tool_calls": tool_calls}) - round_tool_count = 0 - for tc in tool_calls: - func_name = tc.get("function", {}).get("name", "") - check = _check_tool_policy( - func_name, None, skills, tool_policy, tool_call_count, round_tool_count - ) - if not check["allowed"]: - messages.append({ - "role": "tool", - "tool_call_id": tc.get("id", f"call_{round_num}"), - "content": f"[工具调用已被策略拦截] {check.get('message', check.get('reason'))}", - }) - tool_call_count += 1 - round_tool_count += 1 - continue - tool_call_count += 1 - round_tool_count += 1 - try: - func_args = json.loads(tc.get("function", {}).get("arguments", "{}") or "{}") - except Exception: - func_args = {} - res = _execute_skill_tool(db, func_name, func_args, skills, node_input) - called_skills.append(func_name) - messages.append({ - "role": "tool", - "tool_call_id": tc.get("id", f"call_{round_num}"), - "content": res, - }) - return {"text": last_content, "used_tools": called_skills} - - -# --------------------------------------------------------------------------- -# 单节点执行（含独立 DB 会话，保证并发安全） -# --------------------------------------------------------------------------- -async def _run_node(node, graph, blackboard, query, user_id, conv_id, trace_id) -> dict: - from app.api.chat_skill import _resolve_market_agents - from app.database import SessionLocal - - nid = node["id"] - ntype = node["type"] - preds = _predecessors(graph, nid) - pred_results = [blackboard[p] for p in preds] - is_entry = not preds - - if pred_results: - parts = [] - for b in pred_results: - label = f"节点 {b.get('node_id')}/{b.get('node_type')}" - if b.get("ok"): - parts.append(f"【{label}】\n{b.get('output', '')}") - else: - parts.append(f"【{label} 执行失败】\n错误: {b.get('error', '')}") - upstream = "\n\n".join(parts) - else: - upstream = "" - - with SessionLocal() as db: - if ntype in ("llm", "reduce"): - if ntype == "llm": - sys_prompt = node.get("system_prompt") or ( - "你是一个处理节点，请根据上游输入完成指定的文本变换、摘要或路由判断，" - "只输出处理后的结果，不要多余解释。" - ) - else: - sys_prompt = node.get("system_prompt") or ( - "你是一个结果汇总器，请将多个子任务的输出综合成连贯、完整、面向用户的最终回答。" - ) - uc = _assemble_node_input(query, upstream, is_entry) - text = await _llm_complete(sys_prompt, uc, node.get("model_id"), db, user_id) - return {"ok": True, "node_id": nid, "node_type": ntype, "output": text} - - if ntype == "local": - agents = _resolve_market_agents(db, [node["agent_id"]], user_id) - if not agents: - raise OrchestrationError(f"local 节点 {nid} 引用的 agent_id={node['agent_id']} 不存在") - ni = _assemble_node_input(query, upstream, is_entry) - res = await _run_local_agent_turn(agents[0], ni, db, user_id, conv_id) - return { - "ok": True, - "node_id": nid, - "node_type": "local", - "output": res["text"], - "used_tools": res.get("used_tools"), - } - - if ntype == "external": - from app.api.chat_agents import _call_external_agent - - agents = _resolve_market_agents(db, [node["agent_id"]], user_id) - if not agents: - raise OrchestrationError(f"external 节点 {nid} 引用的 agent_id={node['agent_id']} 不存在") - ni = _assemble_node_input(query, upstream, is_entry) - answer, _ = await _call_external_agent( - db, agents[0], query=ni, user_id=user_id, ext_conversation_id=None, history=[] - ) - return {"ok": True, "node_id": nid, "node_type": "external", "output": answer or ""} - - raise OrchestrationError(f"未知节点类型 {ntype}") - - -# --------------------------------------------------------------------------- -# 入口 -# --------------------------------------------------------------------------- -async def run_graph(graph, query, db, user_id, conv_id, trace_id=None): - """执行编排图，返回 (answer: str, trace: dict)。 - - - 拓扑分层后逐层 asyncio.gather 并行执行。 - - 节点异常按 on_error=halt|continue 处理（默认 continue，失败时下游拿到错误作为输入）。 - """ - validate_graph(graph) - layers = topological_layers(graph) - blackboard: dict = {} - trace = {"nodes": [], "trace_id": trace_id, "layers": layers} - - for layer in layers: - tasks = [ - _run_node(_node_by_id(graph, nid), graph, blackboard, query, user_id, conv_id, trace_id) - for nid in layer - ] - results = await asyncio.gather(*tasks, return_exceptions=True) - for nid, res in zip(layer, results): - node = _node_by_id(graph, nid) - if isinstance(res, Exception): - on_error = node.get("on_error", "continue") - err_msg = str(res) - blackboard[nid] = { - "ok": False, - "node_id": nid, - "node_type": node["type"], - "error": err_msg, - } - trace["nodes"].append({"id": nid, "type": node["type"], "ok": False, "error": err_msg}) - logger.warning(f"[orchestrator] 节点 {nid} 失败 on_error={on_error}: {err_msg}") - if on_error == "halt": - raise OrchestrationError(f"节点 {nid} 执行失败(halt): {err_msg}") - else: - blackboard[nid] = res - trace["nodes"].append({ - "id": nid, - "type": res.get("node_type"), - "ok": True, - "used_tools": res.get("used_tools"), - }) - - output_node = _find_output_node(graph) - final = blackboard.get(output_node, {}) - if final.get("ok"): - answer = final.get("output", "") - else: - answer = f"⚠️ 编排输出节点 {output_node} 执行失败: {final.get('error', '未知错误')}" - trace["output_node"] = output_node - return answer, trace +"""app/core/orchestrator.py — Phase 3.3 多 Agent 编排图引擎。 + +设计目标： +- 以轻量 DAG 描述多 Agent 协作：节点(llm/local/external/reduce) + 有向边(数据流)。 +- 拓扑分层后按层并行执行(asyncio.gather)，互相独立的节点真正并发。 +- 失败隔离：每个节点异常不影响其它分支；per-node on_error=halt|continue。 +- 节点执行复用既有原语(_build_system_prompt / _execute_skill_tool / _call_external_agent / + 3.1 工具沙箱)，完全向后兼容——agent.orchestration 为 null 时 chat_service 走原逻辑。 +- 并发安全：每个节点用独立 SessionLocal 会话做 DB 读，避免共享请求会话跨协程竞态。 +""" +from __future__ import annotations + +import asyncio +import json +import logging +from typing import Any, Optional + +import httpx + +logger = logging.getLogger("aiflow.orchestrator") + +VALID_NODE_TYPES = ("llm", "local", "external", "reduce") + + +class OrchestrationError(ValueError): + """图结构非法或执行被 halt。""" + + +# --------------------------------------------------------------------------- +# 校验 + 拓扑 +# --------------------------------------------------------------------------- +def validate_graph(graph: Any) -> None: + if not isinstance(graph, dict): + raise OrchestrationError("orchestration 必须为对象") + nodes = graph.get("nodes") + edges = graph.get("edges") + if not isinstance(nodes, list) or not nodes: + raise OrchestrationError("nodes 必须为非空数组") + if not isinstance(edges, list): + raise OrchestrationError("edges 必须为数组") + + ids = set() + for i, n in enumerate(nodes): + if not isinstance(n, dict): + raise OrchestrationError(f"节点 #{i} 必须为对象") + nid = n.get("id") + if not nid or not isinstance(nid, str): + raise OrchestrationError(f"节点 #{i} 缺少字符串 id") + if nid in ids: + raise OrchestrationError(f"节点 id 重复: {nid}") + ids.add(nid) + ntype = n.get("type") + if ntype not in VALID_NODE_TYPES: + raise OrchestrationError( + f"节点 {nid} 类型非法: {ntype} (允许 {', '.join(VALID_NODE_TYPES)})" + ) + if ntype in ("local", "external") and not n.get("agent_id"): + raise OrchestrationError(f"节点 {nid} 类型 {ntype} 必须指定 agent_id") + + for i, e in enumerate(edges): + if not isinstance(e, dict): + raise OrchestrationError(f"边 #{i} 必须为对象") + f = e.get("from") + t = e.get("to") + if f not in ids: + raise OrchestrationError(f"边 #{i} 的 from={f} 指向不存在的节点") + if t not in ids: + raise OrchestrationError(f"边 #{i} 的 to={t} 指向不存在的节点") + if f == t: + raise OrchestrationError(f"边 #{i} 存在自环: {f}") + + +def topological_layers(graph: dict) -> list: + """Kahn 算法返回分层(每层内节点可并行)；含环则抛错。""" + nodes = graph["nodes"] + edges = graph.get("edges", []) + indeg = {n["id"]: 0 for n in nodes} + adj = {n["id"]: [] for n in nodes} + for e in edges: + f, t = e["from"], e["to"] + adj[f].append(t) + indeg[t] += 1 + order = {n["id"]: i for i, n in enumerate(nodes)} + layers: list = [] + remaining = set(indeg) + while remaining: + layer = sorted([nid for nid in remaining if indeg[nid] == 0], key=lambda x: order[x]) + if not layer: + cycle = sorted(remaining, key=lambda x: order[x]) + raise OrchestrationError(f"检测到环路，无法拓扑排序: {cycle}") + layers.append(layer) + for nid in layer: + remaining.discard(nid) + for nxt in adj[nid]: + indeg[nxt] -= 1 + return layers + + +def _node_by_id(graph: dict, nid: str) -> dict: + for n in graph["nodes"]: + if n["id"] == nid: + return n + return {} + + +def _predecessors(graph: dict, nid: str) -> list: + return [e["from"] for e in graph.get("edges", []) if e["to"] == nid] + + +def _successors(graph: dict, nid: str) -> list: + return [e["to"] for e in graph.get("edges", []) if e["from"] == nid] + + +def _find_output_node(graph: dict) -> str: + if graph.get("output_node"): + return graph["output_node"] + sinks = [n["id"] for n in graph["nodes"] if not _successors(graph, n["id"])] + if len(sinks) == 1: + return sinks[0] + for s in sinks: + if _node_by_id(graph, s).get("type") == "reduce": + return s + if sinks: + return sinks[0] + return graph["nodes"][-1]["id"] + + +def _assemble_node_input(query: str, upstream: str, is_entry: bool) -> str: + """入口节点直接吃用户原问题；下游节点综合原问题 + 上游输出。""" + if is_entry: + return query or "" + if upstream: + return f"原始用户问题:\n{query}\n\n上游节点输出:\n{upstream}" + return query or "" + + +# --------------------------------------------------------------------------- +# 模型解析 + LLM 调用原语 +# --------------------------------------------------------------------------- +async def _resolve_model_and_endpoint(model_id, db): + from app.models import AIModel, ModelProvider + from ..capability_registry import resolve_auto_model + + ai_model = None + if model_id and model_id != "auto": + ai_model = db.query(AIModel).filter(AIModel.id == model_id).first() + if not ai_model: + ai_model = resolve_auto_model("text", db) + if not ai_model: + raise OrchestrationError("无法解析到可用模型") + provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first() + api_key = ai_model.api_key or (provider.api_key if provider else None) + api_url = ai_model.endpoint + if not api_url and provider and provider.api_base_url: + api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions" + if not api_url or not api_key: + raise OrchestrationError(f"节点模型 {ai_model.name} 缺少 endpoint 或 api_key") + return ai_model, api_url, api_key + + +async def _llm_complete(system_prompt: str, user_content: str, model_id, db, user_id) -> str: + from app.api.chat_utils import _proxy_for_url + from app.core.observability import span + + ai_model, api_url, api_key = await _resolve_model_and_endpoint(model_id, db) + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_content}, + ] + request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048} + async with span("orchestrator.llm", model=ai_model.id): + async with httpx.AsyncClient( + timeout=ai_model.timeout or 120, + proxy=_proxy_for_url(api_url), + trust_env=False, + ) as client: + resp = await client.post( + api_url, + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + json=request_body, + ) + if resp.status_code != 200: + raise OrchestrationError(f"LLM 调用失败 HTTP {resp.status_code}: {resp.text[:200]}") + data = resp.json() + return data["choices"][0]["message"].get("content") or "" + + +# --------------------------------------------------------------------------- +# local 节点：复用单 Agent system prompt + 受限 tool round + 3.1 沙箱 +# --------------------------------------------------------------------------- +async def _run_local_agent_turn(agent: dict, node_input: str, db, user_id, conv_id) -> dict: + from app.api.chat_skill import _resolve_market_skills, _execute_skill_tool + from app.api.chat_context import _build_system_prompt, _retrieve_kb_context + from app.api.chat_service import _resolve_agent_tool_policy, _check_tool_policy + from app.api.chat_utils import _proxy_for_url + + skills = _resolve_market_skills(db, agent.get("skill_ids") or [], user_id) + kb_context = await _retrieve_kb_context(db, agent.get("knowledge_base_ids") or [], node_input) + system_prompt, function_definitions = _build_system_prompt( + skills=skills, + agents=[agent], + kb_context=kb_context, + user_content=node_input, + auto_discovered=False, + ) + ai_model, api_url, api_key = await _resolve_model_and_endpoint(agent.get("model_id"), db) + tool_policy = _resolve_agent_tool_policy([agent]) + + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": node_input}, + ] + called_skills: list = [] + tool_call_count = 0 + max_tool_rounds = 5 + last_content = "" + async with httpx.AsyncClient( + timeout=ai_model.timeout or 120, + proxy=_proxy_for_url(api_url), + trust_env=False, + ) as client: + for round_num in range(max_tool_rounds): + request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048} + if function_definitions: + request_body["tools"] = [ + {"type": "function", "function": fd} for fd in function_definitions + ] + request_body["tool_choice"] = "auto" + resp = await client.post( + api_url, + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + json=request_body, + ) + if resp.status_code != 200: + raise OrchestrationError(f"local 节点 LLM 失败 HTTP {resp.status_code}: {resp.text[:200]}") + data = resp.json() + message = data["choices"][0]["message"] + tool_calls = message.get("tool_calls") or [] + content = message.get("content") or "" + last_content = content + if not tool_calls or not function_definitions: + return {"text": content, "used_tools": called_skills} + messages.append({"role": "assistant", "content": content, "tool_calls": tool_calls}) + round_tool_count = 0 + for tc in tool_calls: + func_name = tc.get("function", {}).get("name", "") + check = _check_tool_policy( + func_name, None, skills, tool_policy, tool_call_count, round_tool_count + ) + if not check["allowed"]: + messages.append({ + "role": "tool", + "tool_call_id": tc.get("id", f"call_{round_num}"), + "content": f"[工具调用已被策略拦截] {check.get('message', check.get('reason'))}", + }) + tool_call_count += 1 + round_tool_count += 1 + continue + tool_call_count += 1 + round_tool_count += 1 + try: + func_args = json.loads(tc.get("function", {}).get("arguments", "{}") or "{}") + except Exception: + func_args = {} + res = _execute_skill_tool(db, func_name, func_args, skills, node_input) + called_skills.append(func_name) + messages.append({ + "role": "tool", + "tool_call_id": tc.get("id", f"call_{round_num}"), + "content": res, + }) + return {"text": last_content, "used_tools": called_skills} + + +# --------------------------------------------------------------------------- +# 单节点执行（含独立 DB 会话，保证并发安全） +# --------------------------------------------------------------------------- +async def _run_node(node, graph, blackboard, query, user_id, conv_id, trace_id) -> dict: + from app.api.chat_skill import _resolve_market_agents + from app.database import SessionLocal + + nid = node["id"] + ntype = node["type"] + preds = _predecessors(graph, nid) + pred_results = [blackboard[p] for p in preds] + is_entry = not preds + + if pred_results: + parts = [] + for b in pred_results: + label = f"节点 {b.get('node_id')}/{b.get('node_type')}" + if b.get("ok"): + parts.append(f"【{label}】\n{b.get('output', '')}") + else: + parts.append(f"【{label} 执行失败】\n错误: {b.get('error', '')}") + upstream = "\n\n".join(parts) + else: + upstream = "" + + with SessionLocal() as db: + if ntype in ("llm", "reduce"): + if ntype == "llm": + sys_prompt = node.get("system_prompt") or ( + "你是一个处理节点，请根据上游输入完成指定的文本变换、摘要或路由判断，" + "只输出处理后的结果，不要多余解释。" + ) + else: + sys_prompt = node.get("system_prompt") or ( + "你是一个结果汇总器，请将多个子任务的输出综合成连贯、完整、面向用户的最终回答。" + ) + uc = _assemble_node_input(query, upstream, is_entry) + text = await _llm_complete(sys_prompt, uc, node.get("model_id"), db, user_id) + return {"ok": True, "node_id": nid, "node_type": ntype, "output": text} + + if ntype == "local": + agents = _resolve_market_agents(db, [node["agent_id"]], user_id) + if not agents: + raise OrchestrationError(f"local 节点 {nid} 引用的 agent_id={node['agent_id']} 不存在") + ni = _assemble_node_input(query, upstream, is_entry) + res = await _run_local_agent_turn(agents[0], ni, db, user_id, conv_id) + return { + "ok": True, + "node_id": nid, + "node_type": "local", + "output": res["text"], + "used_tools": res.get("used_tools"), + } + + if ntype == "external": + from app.api.chat_agents import _call_external_agent + + agents = _resolve_market_agents(db, [node["agent_id"]], user_id) + if not agents: + raise OrchestrationError(f"external 节点 {nid} 引用的 agent_id={node['agent_id']} 不存在") + ni = _assemble_node_input(query, upstream, is_entry) + answer, _ = await _call_external_agent( + db, agents[0], query=ni, user_id=user_id, ext_conversation_id=None, history=[] + ) + return {"ok": True, "node_id": nid, "node_type": "external", "output": answer or ""} + + raise OrchestrationError(f"未知节点类型 {ntype}") + + +# --------------------------------------------------------------------------- +# 入口 +# --------------------------------------------------------------------------- +async def run_graph(graph, query, db, user_id, conv_id, trace_id=None): + """执行编排图，返回 (answer: str, trace: dict)。 + + - 拓扑分层后逐层 asyncio.gather 并行执行。 + - 节点异常按 on_error=halt|continue 处理（默认 continue，失败时下游拿到错误作为输入）。 + """ + validate_graph(graph) + layers = topological_layers(graph) + blackboard: dict = {} + trace = {"nodes": [], "trace_id": trace_id, "layers": layers} + + for layer in layers: + tasks = [ + _run_node(_node_by_id(graph, nid), graph, blackboard, query, user_id, conv_id, trace_id) + for nid in layer + ] + results = await asyncio.gather(*tasks, return_exceptions=True) + for nid, res in zip(layer, results): + node = _node_by_id(graph, nid) + if isinstance(res, Exception): + on_error = node.get("on_error", "continue") + err_msg = str(res) + blackboard[nid] = { + "ok": False, + "node_id": nid, + "node_type": node["type"], + "error": err_msg, + } + trace["nodes"].append({"id": nid, "type": node["type"], "ok": False, "error": err_msg}) + logger.warning(f"[orchestrator] 节点 {nid} 失败 on_error={on_error}: {err_msg}") + if on_error == "halt": + raise OrchestrationError(f"节点 {nid} 执行失败(halt): {err_msg}") + else: + blackboard[nid] = res + trace["nodes"].append({ + "id": nid, + "type": res.get("node_type"), + "ok": True, + "used_tools": res.get("used_tools"), + }) + + output_node = _find_output_node(graph) + final = blackboard.get(output_node, {}) + if final.get("ok"): + answer = final.get("output", "") + else: + answer = f"⚠️ 编排输出节点 {output_node} 执行失败: {final.get('error', '未知错误')}" + trace["output_node"] = output_node + return answer, trace diff --git a/backend/python/app/pgvector_search.py b/backend/python/app/pgvector_search.py index a38654a..3f4bf53 100644 --- a/backend/python/app/pgvector_search.py +++ b/backend/python/app/pgvector_search.py @@ -1,360 +1,390 @@ -""" -PGVector 向量检索模块 -==================== -使用 PostgreSQL + pgvector 实现语义向量检索， -与现有 MySQL BM25 检索配合，实现混合检索。 - -使用方式: - from app.pgvector_search import vector_search - results = await vector_search(query, kb_id, top_k=5) -""" - -import os -import logging -import asyncio -from typing import List, Dict, Optional -from datetime import datetime - -logger = logging.getLogger("aiflow.pgvector") - -# PostgreSQL 连接配置 -PG_HOST = os.environ.get("POSTGRES_HOST", "pgvector") -PG_PORT = int(os.environ.get("POSTGRES_PORT", "5432")) -PG_USER = os.environ.get("POSTGRES_USER", "aiflow") -PG_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "pwd2026") -PG_DB = os.environ.get("POSTGRES_DB", "aiflow_vectors") - -# 嵌入模型配置 -EMBEDDING_API_URL = os.environ.get("SILICONFLOW_BASE_URL", "https://api.siliconflow.cn") -EMBEDDING_API_KEY = os.environ.get("SILICONFLOW_API_KEY", "") -EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-large-zh-v1.5") - -# 连接池 -_pg_pool = None -_pool_lock = asyncio.Lock() - - -async def get_pg_pool(): - """获取 PostgreSQL 异步连接池""" - global _pg_pool - if _pg_pool is not None: - return _pg_pool - - async with _pool_lock: - if _pg_pool is not None: - return _pg_pool - - try: - import asyncpg - except ImportError: - logger.warning("asyncpg not installed, vector search disabled") - return None - - try: - _pg_pool = await asyncpg.create_pool( - host=PG_HOST, - port=PG_PORT, - user=PG_USER, - password=PG_PASSWORD, - database=PG_DB, - min_size=2, - max_size=10, - command_timeout=30, - ) - logger.info(f"PostgreSQL connection pool created: {PG_HOST}:{PG_PORT}/{PG_DB}") - - # 初始化表结构 - async with _pg_pool.acquire() as conn: - await conn.execute(""" - CREATE TABLE IF NOT EXISTS document_embeddings ( - id SERIAL PRIMARY KEY, - doc_id VARCHAR(64) NOT NULL, - kb_id VARCHAR(64) NOT NULL, - chunk_id VARCHAR(64), - content TEXT NOT NULL, - embedding vector(1024), - metadata JSONB DEFAULT '{}', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - """) - await conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_embeddings_kb - ON document_embeddings(kb_id); - """) - await conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_embeddings_doc - ON document_embeddings(doc_id); - """) - await conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_embeddings_vector - ON document_embeddings USING ivfflat (embedding vector_cosine_ops) - WITH (lists = 100); - """) - logger.info("PGVector tables initialized") - - return _pg_pool - except Exception as e: - logger.error(f"Failed to create PostgreSQL pool: {e}") - _pg_pool = None - return None - - -async def get_embedding(text: str) -> Optional[List[float]]: - """调用嵌入API获取文本向量""" - if not EMBEDDING_API_KEY: - logger.warning("No embedding API key configured") - return None - - try: - import httpx - async with httpx.AsyncClient(timeout=30) as client: - resp = await client.post( - f"{EMBEDDING_API_URL}/v1/embeddings", - headers={ - "Authorization": f"Bearer {EMBEDDING_API_KEY}", - "Content-Type": "application/json", - }, - json={ - "model": EMBEDDING_MODEL, - "input": text[:8000], # 截断防止超长 - }, - ) - if resp.status_code == 200: - data = resp.json() - embedding = data["data"][0]["embedding"] - return embedding - else: - logger.error(f"Embedding API error: {resp.status_code} {resp.text[:200]}") - return None - except Exception as e: - logger.error(f"Embedding generation failed: {e}") - return None - - -async def vector_search( - query: str, - kb_id: Optional[str] = None, - top_k: int = 5, - threshold: float = 0.5, -) -> List[Dict]: - """ - 向量语义检索 - - Args: - query: 查询文本 - kb_id: 知识库ID (可选，不指定则搜索全部) - top_k: 返回结果数 - threshold: 相似度阈值 (0-1) - - Returns: - [{"doc_id": ..., "content": ..., "score": ..., "metadata": ...}, ...] - """ - pool = await get_pg_pool() - if pool is None: - return [] - - # 获取查询向量 - query_embedding = await get_embedding(query) - if query_embedding is None: - return [] - - try: - async with pool.acquire() as conn: - # 将向量转换为 PostgreSQL 格式 - embedding_str = f"[{','.join(str(x) for x in query_embedding)}]" - - if kb_id: - rows = await conn.fetch( - """ - SELECT doc_id, chunk_id, content, metadata, - 1 - (embedding <=> $1::vector) as score - FROM document_embeddings - WHERE kb_id = $2 - ORDER BY embedding <=> $1::vector - LIMIT $3; - """, - embedding_str, kb_id, top_k, - ) - else: - rows = await conn.fetch( - """ - SELECT doc_id, chunk_id, content, metadata, - 1 - (embedding <=> $1::vector) as score - FROM document_embeddings - ORDER BY embedding <=> $1::vector - LIMIT $2; - """, - embedding_str, top_k, - ) - - results = [] - for row in rows: - score = float(row["score"]) - if score >= threshold: - results.append({ - "doc_id": row["doc_id"], - "chunk_id": row["chunk_id"], - "content": row["content"], - "score": score, - "metadata": row["metadata"] if isinstance(row["metadata"], dict) else {}, - }) - - return results - - except Exception as e: - logger.error(f"Vector search failed: {e}") - return [] - - -async def index_document( - doc_id: str, - kb_id: str, - content: str, - chunk_id: Optional[str] = None, - metadata: Optional[Dict] = None, -) -> bool: - """ - 将文档内容索引到 PGVector - - Args: - doc_id: 文档ID - kb_id: 知识库ID - content: 文档内容 - chunk_id: 分块ID (可选) - metadata: 元数据 - - Returns: - 是否成功 - """ - pool = await get_pg_pool() - if pool is None: - return False - - # 分块处理（简单按长度分块） - chunks = [] - chunk_size = 500 - overlap = 50 - if len(content) <= chunk_size: - chunks = [(content, chunk_id or f"{doc_id}_0")] - else: - for i in range(0, len(content), chunk_size - overlap): - chunk = content[i:i + chunk_size] - if len(chunk.strip()) > 10: - chunks.append((chunk, f"{doc_id}_{i // (chunk_size - overlap)}")) - - success_count = 0 - for chunk_text, cid in chunks: - embedding = await get_embedding(chunk_text) - if embedding is None: - continue - - try: - async with pool.acquire() as conn: - embedding_str = f"[{','.join(str(x) for x in embedding)}]" - import json - await conn.execute( - """ - INSERT INTO document_embeddings (doc_id, kb_id, chunk_id, content, embedding, metadata) - VALUES ($1, $2, $3, $4, $5::vector, $6) - ON CONFLICT DO NOTHING; - """, - doc_id, kb_id, cid, chunk_text, embedding_str, - json.dumps(metadata or {}), - ) - success_count += 1 - except Exception as e: - logger.error(f"Failed to index chunk {cid}: {e}") - - logger.info(f"Indexed document {doc_id}: {success_count}/{len(chunks)} chunks") - return success_count > 0 - - -async def hybrid_search( - query: str, - kb_id: Optional[str] = None, - top_k: int = 5, - bm25_results: Optional[List[Dict]] = None, -) -> List[Dict]: - """ - 混合检索：BM25 + 向量检索融合 - - Args: - query: 查询文本 - kb_id: 知识库ID - top_k: 返回结果数 - bm25_results: BM25 检索结果（来自MySQL） - - Returns: - 融合后的检索结果 - """ - # 获取向量检索结果 - vec_results = await vector_search(query, kb_id, top_k) - - if not vec_results and not bm25_results: - return [] - - if not vec_results: - return bm25_results or [] - - if not bm25_results: - return vec_results - - # 融合：Reciprocal Rank Fusion (RRF) - rrf_k = 60 - score_map = {} - - for rank, item in enumerate(bm25_results): - doc_id = item.get("doc_id", item.get("id", "")) - score_map[doc_id] = { - "content": item.get("content", item.get("text", "")), - "score": 1.0 / (rrf_k + rank + 1), - "bm25_rank": rank + 1, - "metadata": item.get("metadata", {}), - } - - for rank, item in enumerate(vec_results): - doc_id = item["doc_id"] - if doc_id in score_map: - score_map[doc_id]["score"] += 1.0 / (rrf_k + rank + 1) - score_map[doc_id]["vec_score"] = item["score"] - score_map[doc_id]["vec_rank"] = rank + 1 - else: - score_map[doc_id] = { - "content": item["content"], - "score": 1.0 / (rrf_k + rank + 1), - "vec_score": item["score"], - "vec_rank": rank + 1, - "metadata": item.get("metadata", {}), - } - - # 按融合分数排序 - merged = sorted(score_map.values(), key=lambda x: x["score"], reverse=True) - return merged[:top_k] - - -async def delete_document_embeddings(doc_id: str) -> bool: - """删除文档的所有向量索引""" - pool = await get_pg_pool() - if pool is None: - return False - - try: - async with pool.acquire() as conn: - await conn.execute( - "DELETE FROM document_embeddings WHERE doc_id = $1;", - doc_id, - ) - logger.info(f"Deleted embeddings for document {doc_id}") - return True - except Exception as e: - logger.error(f"Failed to delete embeddings: {e}") - return False - - -async def close_pool(): - """关闭连接池""" - global _pg_pool - if _pg_pool is not None: - await _pg_pool.close() - _pg_pool = None - logger.info("PostgreSQL connection pool closed") +""" +PGVector 向量检索模块 +==================== +使用 PostgreSQL + pgvector 实现语义向量检索， +与现有 MySQL BM25 检索配合，实现混合检索。 + +使用方式: + from app.pgvector_search import vector_search + results = await vector_search(query, kb_id, top_k=5) +""" + +import os +import logging +import asyncio +from typing import List, Dict, Optional +from datetime import datetime + +logger = logging.getLogger("aiflow.pgvector") + +# PostgreSQL 连接配置 +PG_HOST = os.environ.get("POSTGRES_HOST", "qihu-aiflow-pgvector") +PG_PORT = int(os.environ.get("POSTGRES_PORT", "5432")) +PG_USER = os.environ.get("POSTGRES_USER", "aiflow") +PG_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "pwd2026") +PG_DB = os.environ.get("POSTGRES_DB", "aiflow_vectors") + +# 嵌入模型配置 +EMBEDDING_API_URL = os.environ.get("SILICONFLOW_BASE_URL", "https://api.siliconflow.cn") +EMBEDDING_API_KEY = os.environ.get("SILICONFLOW_API_KEY", "") +EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-large-zh-v1.5") + +# ---- 本地嵌入缓存（SQLite 持久化）：破解"每次嵌入消耗外部 API 额度"的隐性成本障碍 ---- +import sqlite3, hashlib, json as _json +_EMBED_CACHE_DB = os.environ.get("EMBED_CACHE_DB", "/app/backend/python/.embed_cache.db") +def _embed_cache_get(text, model): + try: + key = hashlib.sha256(f"{model}::{text}".encode("utf-8")).hexdigest() + conn = sqlite3.connect(_EMBED_CACHE_DB, timeout=5) + row = conn.execute("SELECT vec FROM embed_cache WHERE k=?", (key,)).fetchone() + conn.close() + if row: + return _json.loads(row[0]) + except Exception: + return None + return None +def _embed_cache_put(text, model, vec): + try: + key = hashlib.sha256(f"{model}::{text}".encode("utf-8")).hexdigest() + conn = sqlite3.connect(_EMBED_CACHE_DB, timeout=5) + conn.execute("CREATE TABLE IF NOT EXISTS embed_cache (k TEXT PRIMARY KEY, vec TEXT, ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP)") + conn.execute("INSERT OR REPLACE INTO embed_cache (k, vec) VALUES (?, ?)", (key, _json.dumps(vec))) + conn.commit() + conn.close() + except Exception: + pass + + +# 连接池 +_pg_pool = None +_pool_lock = asyncio.Lock() + + +async def get_pg_pool(): + """获取 PostgreSQL 异步连接池""" + global _pg_pool + if _pg_pool is not None: + return _pg_pool + + async with _pool_lock: + if _pg_pool is not None: + return _pg_pool + + try: + import asyncpg + except ImportError: + logger.warning("asyncpg not installed, vector search disabled") + return None + + try: + _pg_pool = await asyncpg.create_pool( + host=PG_HOST, + port=PG_PORT, + user=PG_USER, + password=PG_PASSWORD, + database=PG_DB, + min_size=2, + max_size=10, + command_timeout=30, + ) + logger.info(f"PostgreSQL connection pool created: {PG_HOST}:{PG_PORT}/{PG_DB}") + + # 初始化表结构 + async with _pg_pool.acquire() as conn: + await conn.execute(""" + CREATE TABLE IF NOT EXISTS document_embeddings ( + id SERIAL PRIMARY KEY, + doc_id VARCHAR(64) NOT NULL, + kb_id VARCHAR(64) NOT NULL, + chunk_id VARCHAR(64), + content TEXT NOT NULL, + embedding vector(1024), + metadata JSONB DEFAULT '{}', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """) + await conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_embeddings_kb + ON document_embeddings(kb_id); + """) + await conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_embeddings_doc + ON document_embeddings(doc_id); + """) + await conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_embeddings_vector + ON document_embeddings USING ivfflat (embedding vector_cosine_ops) + WITH (lists = 100); + """) + logger.info("PGVector tables initialized") + + return _pg_pool + except Exception as e: + logger.error(f"Failed to create PostgreSQL pool: {e}") + _pg_pool = None + return None + + +async def get_embedding(text: str) -> Optional[List[float]]: + """调用嵌入API获取文本向量""" + if not EMBEDDING_API_KEY: + logger.warning("No embedding API key configured") + return None + + cached = _embed_cache_get(text, EMBEDDING_MODEL) + if cached: + return cached + try: + import httpx + async with httpx.AsyncClient(timeout=30, trust_env=False) as client: + resp = await client.post( + f"{EMBEDDING_API_URL}/v1/embeddings", + headers={ + "Authorization": f"Bearer {EMBEDDING_API_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": EMBEDDING_MODEL, + "input": text[:8000], # 截断防止超长 + }, + ) + if resp.status_code == 200: + data = resp.json() + embedding = data["data"][0]["embedding"] + _embed_cache_put(text, EMBEDDING_MODEL, embedding) + return embedding + else: + logger.error(f"Embedding API error: {resp.status_code} {resp.text[:200]}") + return None + except Exception as e: + logger.error(f"Embedding generation failed: {e}") + return None + + +async def vector_search( + query: str, + kb_id: Optional[str] = None, + top_k: int = 5, + threshold: float = 0.5, +) -> List[Dict]: + """ + 向量语义检索 + + Args: + query: 查询文本 + kb_id: 知识库ID (可选，不指定则搜索全部) + top_k: 返回结果数 + threshold: 相似度阈值 (0-1) + + Returns: + [{"doc_id": ..., "content": ..., "score": ..., "metadata": ...}, ...] + """ + pool = await get_pg_pool() + if pool is None: + return [] + + # 获取查询向量 + query_embedding = await get_embedding(query) + if query_embedding is None: + return [] + + try: + async with pool.acquire() as conn: + # 将向量转换为 PostgreSQL 格式 + embedding_str = f"[{','.join(str(x) for x in query_embedding)}]" + + if kb_id: + rows = await conn.fetch( + """ + SELECT doc_id, chunk_id, content, metadata, + 1 - (embedding <=> $1::vector) as score + FROM document_embeddings + WHERE kb_id = $2 + ORDER BY embedding <=> $1::vector + LIMIT $3; + """, + embedding_str, kb_id, top_k, + ) + else: + rows = await conn.fetch( + """ + SELECT doc_id, chunk_id, content, metadata, + 1 - (embedding <=> $1::vector) as score + FROM document_embeddings + ORDER BY embedding <=> $1::vector + LIMIT $2; + """, + embedding_str, top_k, + ) + + results = [] + for row in rows: + score = float(row["score"]) + if score >= threshold: + results.append({ + "doc_id": row["doc_id"], + "chunk_id": row["chunk_id"], + "content": row["content"], + "score": score, + "metadata": row["metadata"] if isinstance(row["metadata"], dict) else {}, + }) + + return results + + except Exception as e: + logger.error(f"Vector search failed: {e}") + return [] + + +async def index_document( + doc_id: str, + kb_id: str, + content: str, + chunk_id: Optional[str] = None, + metadata: Optional[Dict] = None, +) -> bool: + """ + 将文档内容索引到 PGVector + + Args: + doc_id: 文档ID + kb_id: 知识库ID + content: 文档内容 + chunk_id: 分块ID (可选) + metadata: 元数据 + + Returns: + 是否成功 + """ + pool = await get_pg_pool() + if pool is None: + return False + + # 分块处理（简单按长度分块） + chunks = [] + chunk_size = 500 + overlap = 50 + if len(content) <= chunk_size: + chunks = [(content, chunk_id or f"{doc_id}_0")] + else: + for i in range(0, len(content), chunk_size - overlap): + chunk = content[i:i + chunk_size] + if len(chunk.strip()) > 10: + chunks.append((chunk, f"{doc_id}_{i // (chunk_size - overlap)}")) + + success_count = 0 + for chunk_text, cid in chunks: + embedding = await get_embedding(chunk_text) + if embedding is None: + continue + + try: + async with pool.acquire() as conn: + embedding_str = f"[{','.join(str(x) for x in embedding)}]" + import json + await conn.execute( + """ + INSERT INTO document_embeddings (doc_id, kb_id, chunk_id, content, embedding, metadata) + VALUES ($1, $2, $3, $4, $5::vector, $6) + ON CONFLICT DO NOTHING; + """, + doc_id, kb_id, cid, chunk_text, embedding_str, + json.dumps(metadata or {}), + ) + success_count += 1 + except Exception as e: + logger.error(f"Failed to index chunk {cid}: {e}") + + logger.info(f"Indexed document {doc_id}: {success_count}/{len(chunks)} chunks") + return success_count > 0 + + +async def hybrid_search( + query: str, + kb_id: Optional[str] = None, + top_k: int = 5, + bm25_results: Optional[List[Dict]] = None, +) -> List[Dict]: + """ + 混合检索：BM25 + 向量检索融合 + + Args: + query: 查询文本 + kb_id: 知识库ID + top_k: 返回结果数 + bm25_results: BM25 检索结果（来自MySQL） + + Returns: + 融合后的检索结果 + """ + # 获取向量检索结果 + vec_results = await vector_search(query, kb_id, top_k) + + if not vec_results and not bm25_results: + return [] + + if not vec_results: + return bm25_results or [] + + if not bm25_results: + return vec_results + + # 融合：Reciprocal Rank Fusion (RRF) + rrf_k = 60 + score_map = {} + + for rank, item in enumerate(bm25_results): + doc_id = item.get("doc_id", item.get("id", "")) + score_map[doc_id] = { + "content": item.get("content", item.get("text", "")), + "score": 1.0 / (rrf_k + rank + 1), + "bm25_rank": rank + 1, + "metadata": item.get("metadata", {}), + } + + for rank, item in enumerate(vec_results): + doc_id = item["doc_id"] + if doc_id in score_map: + score_map[doc_id]["score"] += 1.0 / (rrf_k + rank + 1) + score_map[doc_id]["vec_score"] = item["score"] + score_map[doc_id]["vec_rank"] = rank + 1 + else: + score_map[doc_id] = { + "content": item["content"], + "score": 1.0 / (rrf_k + rank + 1), + "vec_score": item["score"], + "vec_rank": rank + 1, + "metadata": item.get("metadata", {}), + } + + # 按融合分数排序 + merged = sorted(score_map.values(), key=lambda x: x["score"], reverse=True) + return merged[:top_k] + + +async def delete_document_embeddings(doc_id: str) -> bool: + """删除文档的所有向量索引""" + pool = await get_pg_pool() + if pool is None: + return False + + try: + async with pool.acquire() as conn: + await conn.execute( + "DELETE FROM document_embeddings WHERE doc_id = $1;", + doc_id, + ) + logger.info(f"Deleted embeddings for document {doc_id}") + return True + except Exception as e: + logger.error(f"Failed to delete embeddings: {e}") + return False + + +async def close_pool(): + """关闭连接池""" + global _pg_pool + if _pg_pool is not None: + await _pg_pool.close() + _pg_pool = None + logger.info("PostgreSQL connection pool closed") diff --git a/backend/python/app/platform_knowledge.py b/backend/python/app/platform_knowledge.py index a364fcf..3ec588b 100644 --- a/backend/python/app/platform_knowledge.py +++ b/backend/python/app/platform_knowledge.py @@ -81,6 +81,41 @@ PLATFORM_KNOWLEDGE = """# 七虎AI平台能力全览 """ -def get_platform_knowledge() -> str: - """获取平台全功能知识库文本，注入到AI对话系统提示中""" - return PLATFORM_KNOWLEDGE +def get_platform_knowledge(db=None, user_id=None) -> str: + """获取平台全功能知识库文本，注入到AI对话系统提示中。 + + 若传入 db，则动态追加当前平台已上线的官方技能目录 + （source=official 且 status=published），确保管理员/前台导入或下架技能后， + AI 能及时感知并使用「平台自身已有内容」，而非停留在陈旧硬编码清单（见 #712）。 + 任何异常均安全回退到静态 PLATFORM_KNOWLEDGE。 + """ + if db is None: + return PLATFORM_KNOWLEDGE + try: + from .models import MarketApp + + apps = ( + db.query(MarketApp) + .filter(MarketApp.source == "official", MarketApp.status == "published") + .order_by(MarketApp.capability, MarketApp.name) + .all() + ) + if not apps: + return PLATFORM_KNOWLEDGE + + lines = [ + "", + "## 平台已上线技能（官方，优先推荐）", + "以下为平台当前已发布上线的技能/应用，回答用户相关问题时**优先推荐使用**这些平台能力：", + ] + for a in apps: + name = getattr(a, "name", None) or "未命名" + cap = getattr(a, "capability", None) or getattr(a, "category", None) or "通用" + desc = (getattr(a, "description", "") or "").strip().replace("\n", " ") + if len(desc) > 60: + desc = desc[:60] + "…" + lines.append(f"- **{name}**（{cap}）: {desc}") + return PLATFORM_KNOWLEDGE + "\n" + "\n".join(lines) + except Exception as e: + logger.warning(f"[platform_knowledge] 动态构建失败，回退静态知识库: {e}") + return PLATFORM_KNOWLEDGE diff --git a/frontend/src/pages/admin/Skills.tsx b/frontend/src/pages/admin/Skills.tsx index 274cfc3..c801055 100644 --- a/frontend/src/pages/admin/Skills.tsx +++ b/frontend/src/pages/admin/Skills.tsx @@ -1,676 +1,682 @@ -import React, { useState, useEffect } from 'react'; -import { adminAPI } from '@/api'; -import type { SkillState } from '@/types'; - -type AdminSkill = { - id: string; - user_id: string; - owner_username?: string; - skill_name: string; - description?: string; - tags: string[]; - category?: string; - category_label?: string; - state: SkillState; - use_count: number; - version: number; - skill_md_path?: string; - last_used_at?: string; - created_at?: string; - updated_at?: string; - skill_md_content?: string; - publish_records?: PublishRecord[]; -}; - -interface PublishRecord { - id: string; - price: number; - status: string; - published_at?: string; -} - -interface SkillVersion { - id: string; - version: number; - description?: string; - skill_md_content?: string; - created_at?: string; -} - -interface SkillsStats { - total_skills: number; - by_state: Record; - by_category: Record; - total_published: number; - total_versions: number; -} - -const STATE_CONFIG: Record = { - baby: { label: '萌芽期', color: '#6B7280', bg: '#F3F4F6' }, - main: { label: '主力期', color: '#10B981', bg: '#D1FAE5' }, - staging: { label: '灰度期', color: '#F59E0B', bg: '#FEF3C7' }, - frozen: { label: '已冻结', color: '#EF4444', bg: '#FEE2E2' }, -}; - -const CATEGORIES = [ - { id: 'writing', label: '文字创作' }, - { id: 'coding', label: '代码能力' }, - { id: 'data_analysis', label: '数据分析' }, - { id: 'qa', label: '知识问答' }, - { id: 'automation', label: '流程自动化' }, - { id: 'creativity', label: '创意生成' }, - { id: 'communication', label: '沟通协作' }, - { id: 'reasoning', label: '逻辑推理' }, - { id: 'other', label: '其他' }, -]; - -const PAGE_SIZE = 15; - -const Skills: React.FC = () => { - const [skills, setSkills] = useState([]); - const [total, setTotal] = useState(0); - const [page, setPage] = useState(1); - const [loading, setLoading] = useState(false); - const [stats, setStats] = useState(null); - - // Filters - const [keyword, setKeyword] = useState(''); - const [filterState, setFilterState] = useState(''); - const [filterCategory, setFilterCategory] = useState(''); - - // Create/Edit Modal - const [showModal, setShowModal] = useState(false); - const [editingSkill, setEditingSkill] = useState(null); - const [form, setForm] = useState({ - skill_name: '', - user_id: '', - description: '', - tags: '', - category: '', - skill_md_content: '## 技能定义\n\n### 触发词\n- /skill\n\n### 能力说明\n请描述这个技能的核心能力...\n\n### 使用场景\n1. 场景一\n2. 场景二\n\n### 示例对话\n**用户**: \n**AI**: ', - }); - - // Version Drawer - const [showVersions, setShowVersions] = useState(false); - const [versions, setVersions] = useState([]); - const [versionSkill, setVersionSkill] = useState(null); - const [loadingVersions, setLoadingVersions] = useState(false); - - // Users list (for owner selection) - const [users, setUsers] = useState<{id: string; username: string}[]>([]); - - // Confirm delete - const [deleteTarget, setDeleteTarget] = useState(null); - - // Toast - const [toast, setToast] = useState<{msg: string; type: 'success' | 'error'} | null>(null); - - const showToast = (msg: string, type: 'success' | 'error' = 'success') => { - setToast({ msg, type }); - setTimeout(() => setToast(null), 3000); - }; - - // Load skills list - const loadSkills = async () => { - setLoading(true); - try { - const params: any = { page, page_size: PAGE_SIZE }; - if (keyword) params.keyword = keyword; - if (filterState) params.state = filterState; - if (filterCategory) params.category = filterCategory; - - const result = await adminAPI.getAdminSkills(params) as any; - const data = result.data || result; - setSkills(data.items || []); - setTotal(data.total || 0); - } catch (e: any) { - showToast(`加载失败: ${e.message || e}`, 'error'); - } finally { - setLoading(false); - } - }; - - // Load stats - const loadStats = async () => { - try { - const result = await adminAPI.getAdminSkillsStats() as any; - const data = result.data || result; - setStats(data); - } catch {} - }; - - // Load users for owner selection - const loadUsers = async () => { - try { - const result = await adminAPI.getUsers({ page_size: 100 }) as any; - const data = result.data || result; - setUsers((data.items || data || []).slice(0, 20)); - } catch {} - }; - - useEffect(() => { - loadSkills(); - }, [page, keyword, filterState, filterCategory]); - - useEffect(() => { - loadStats(); - loadUsers(); - }, []); - - // Search - const handleSearch = () => { - setPage(1); - loadSkills(); - }; - - // Open create modal - const openCreate = () => { - setEditingSkill(null); - setForm({ - skill_name: '', - user_id: users[0]?.id || '', - description: '', - tags: '', - category: '', - skill_md_content: '## 技能定义\n\n### 触发词\n- /skill\n\n### 能力说明\n请描述这个技能的核心能力...\n\n### 使用场景\n1. 场景一\n2. 场景二\n\n### 示例对话\n**用户**: \n**AI**: ', - }); - setShowModal(true); - }; - - // Open edit modal - const openEdit = (skill: AdminSkill) => { - setEditingSkill(skill); - setForm({ - skill_name: skill.skill_name, - user_id: skill.user_id, - description: skill.description || '', - tags: (skill.tags || []).join(', '), - category: skill.category || '', - skill_md_content: skill.skill_md_content || '', - }); - setShowModal(true); - }; - - // Save skill - const handleSave = async () => { - if (!form.skill_name.trim()) { - showToast('技能名称不能为空', 'error'); - return; - } - if (!form.user_id) { - showToast('请选择所属用户', 'error'); - return; - } - - const tags = form.tags.split(',').map(t => t.trim()).filter(Boolean); - const payload: any = { - skill_name: form.skill_name, - user_id: form.user_id, - description: form.description, - tags, - category: form.category || undefined, - skill_md_content: form.skill_md_content, - }; - - try { - if (editingSkill) { - await adminAPI.updateAdminSkill(editingSkill.id, payload); - showToast('技能已更新'); - } else { - await adminAPI.createAdminSkill(payload); - showToast('技能已创建'); - } - setShowModal(false); - loadSkills(); - loadStats(); - } catch (e: any) { - showToast(`保存失败: ${e.message || e}`, 'error'); - } - }; - - // Delete skill - const handleDelete = async () => { - if (!deleteTarget) return; - try { - await adminAPI.deleteAdminSkill(deleteTarget.id); - showToast('技能已删除'); - setDeleteTarget(null); - loadSkills(); - loadStats(); - } catch (e: any) { - showToast(`删除失败: ${e.message || e}`, 'error'); - } - }; - - // Change state - const handleChangeState = async (skill: AdminSkill, newState: string) => { - try { - await adminAPI.changeAdminSkillState(skill.id, { state: newState }); - showToast(`状态已变更为 ${STATE_CONFIG[newState as SkillState].label}`); - loadSkills(); - loadStats(); - } catch (e: any) { - showToast(`状态修改失败: ${e.message || e}`, 'error'); - } - }; - - // Load version history - const handleShowVersions = async (skill: AdminSkill) => { - setVersionSkill(skill); - setShowVersions(true); - setLoadingVersions(true); - try { - const result = await adminAPI.getAdminSkillVersions(skill.id) as any; - const data = result.data || result; - setVersions(data.versions || []); - } catch (e: any) { - showToast(`加载版本历史失败: ${e.message || e}`, 'error'); - setVersions([]); - } finally { - setLoadingVersions(false); - } - }; - - // Rollback version - const handleRollback = async (skill: AdminSkill, targetVersion: number) => { - if (!confirm(`确定回滚到版本 ${targetVersion} 吗？`)) return; - try { - await adminAPI.rollbackAdminSkill(skill.id, { version: targetVersion }); - showToast(`已回滚到版本 ${targetVersion}`); - setShowVersions(false); - loadSkills(); - } catch (e: any) { - showToast(`回滚失败: ${e.message || e}`, 'error'); - } - }; - - // Publish/Unpublish - const handlePublish = async (skill: AdminSkill) => { - const price = parseFloat(prompt('设置定价（元，0=免费）:', '0') || '0'); - if (isNaN(price) || price < 0) return; - try { - await adminAPI.publishAdminSkill(skill.id, { price, market_description: skill.description || '' }); - showToast('已发布到应用市场'); - loadSkills(); - } catch (e: any) { - showToast(`发布失败: ${e.message || e}`, 'error'); - } - }; - - const handleUnpublish = async (skill: AdminSkill) => { - if (!confirm('确定从市场下架吗？')) return; - try { - await adminAPI.unpublishAdminSkill(skill.id); - showToast('已从市场下架'); - loadSkills(); - } catch (e: any) { - showToast(`下架失败: ${e.message || e}`, 'error'); - } - }; - - const totalPages = Math.ceil(total / PAGE_SIZE); - const isPublished = (skill: AdminSkill) => - (skill.publish_records || []).some(r => r.status === 'published'); - - return ( -
- {/* Toast */} - {toast && ( -
- {toast.msg} -
- )} - - {/* Header */} -
-
-

Skills技能管理

-

- 管理全平台 AI 技能，支持版本历史与市场发布 -

-
-
- - {/* Stats Cards */} - {stats && ( -
-
-
{stats.total_skills}
-
技能总数
-
- {Object.entries(STATE_CONFIG).map(([state, cfg]) => ( -
-
{stats.by_state[state] || 0}
-
{cfg.label}
-
- ))} -
-
{stats.total_published}
-
已发布市场
-
-
-
{stats.total_versions}
-
版本记录
-
-
- )} - - {/* Filters */} -
- setKeyword(e.target.value)} - onKeyDown={e => e.key === 'Enter' && handleSearch()} - style={{ maxWidth: 280 }} - /> - - - - {(keyword || filterState || filterCategory) && ( - - )} -
- - {/* Table */} -
- - - - - - - - - - - - - - - - - {loading ? ( - - ) : skills.length === 0 ? ( - - ) : skills.map(skill => { - const cfg = STATE_CONFIG[skill.state as SkillState] || STATE_CONFIG.baby; - const published = isPublished(skill); - return ( - - - - - - - - - - - - - ); - })} - -
技能名称状态分类标签所有者版本使用次数市场更新时间操作
加载中...
暂无技能
-
{skill.skill_name}
- {skill.description && ( -
- {skill.description} -
- )} - {skill.tags && skill.tags.length > 0 && ( -
- {skill.tags.slice(0, 3).map(tag => ( - {tag} - ))} - {skill.tags.length > 3 && +{skill.tags.length - 3}} -
- )} - 		- - {cfg.label} - - {skill.category_label || '-'} - {skill.tags && skill.tags.length > 0 ? ( -
- {skill.tags.slice(0, 3).map(tag => ( - {tag} - ))} - {skill.tags.length > 3 && +{skill.tags.length - 3}} -
- ) : -} - 		{skill.owner_username || skill.user_id}v{skill.version}{skill.use_count} - {published ? ( - ✓ 已发布 - ) : ( - 未发布 - )} - - {skill.updated_at ? skill.updated_at.slice(0, 16).replace('T', ' ') : '-'} - -
- - - {published ? ( - - ) : ( - - )} - {/* State change */} - - -
-
-
- - {/* Pagination */} - {totalPages > 1 && ( -
- - 共 {total} 条，第 {page}/{totalPages} 页 - -
- - {[...Array(Math.min(5, totalPages))].map((_, i) => { - const p = Math.max(1, Math.min(page - 2, totalPages - 4)) + i; - return p <= totalPages && ( - - ); - })} - -
-
- )} - - {/* Create/Edit Modal */} - {showModal && ( -
setShowModal(false)}> -
e.stopPropagation()}> -
-

{editingSkill ? '编辑技能' : '创建技能'}

- -
-
-
-
- - setForm(f => ({ ...f, skill_name: e.target.value }))} - placeholder="例如: 代码审查助手" /> -
- {!editingSkill && ( -
- - -
- )} -
- - -
-
- - setForm(f => ({ ...f, tags: e.target.value }))} - placeholder="例如: 代码, 审查, 自动化" /> -
-
- -