qihuaiflow/backend/python/app/api/chat_skill.py

# Auto-extracted from chat.py during Phase 1 refactor (Strangler Fig).
# Source of truth: scripts/phase1_extract.py. Do not hand-merge back.

from app.core.credential import decrypt_secret, mask_secret
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy.orm import Session
from sqlalchemy import and_, desc, asc, func
from pydantic import BaseModel, Field
from typing import Optional
import uuid
from datetime import datetime, timezone, timedelta
import re
import json
import asyncio
import math
import httpx
import os
import logging
from ..database import get_db, SessionLocal
from ..models import (



    User, Conversation, Message, AIModel, ModelProvider, TokenUsageLog,



    KnowledgeBase, KnowledgeDocument, KnowledgeDocumentChunk, EvolutionSkill,



    EvolutionTrajectory, MarketApp, AppPurchase, AppInstallation,



    MCPServer,



    gen_uuid



)
from ..auth import get_current_user
from ..platform_knowledge import get_platform_knowledge
from ..mcp_client import list_mcp_tools, call_mcp_tool
from ..middleware.log_service import LogService
from ..capability_registry import resolve_auto_model

def _resolve_market_skills(db: Session, skill_ids: list, user_id: str) -> list:



    """



    Stage A.1: 统一Skill ID空间



    前端传MarketApp.id, 需要映射为可用的技能定义。



    返回: [{id, name, description, skill_content, category, func_name}]



    """



    if not skill_ids:



        return []







    # 1. 从MarketApp查询 (前端传的是MarketApp.id)



    market_skills = db.query(MarketApp).filter(



        MarketApp.id.in_(skill_ids),



        MarketApp.type == "skill"



    ).all()







    results = []



    for ms in market_skills:



        skill_content = ms.skill_content or ""



        # 尝试从skill_content解析SKILL.md格式



        # SKILL.md格式: ---\nname: ...\ndescription: ...\n---\n内容...



        parsed = _parse_skill_content(skill_content)







        func_name = ms.name.replace(" ", "_").replace("-", "_").lower()



        results.append({



            "id": ms.id,



            "market_app_id": ms.id,



            "name": ms.name,



            "description": ms.description or parsed.get("description", ""),



            "skill_content": skill_content,



            "category": parsed.get("category", "general"),



            "skill_type": parsed.get("skill_type", "general"),



            "func_name": func_name,



            "parameters": parsed.get("parameters", {"input": {"type": "string", "description": "用户输入内容"}}),



            "instructions": parsed.get("instructions", ""),



        })







    # 2. 如果有未匹配的ID, 尝试查EvolutionSkill (兼容旧的进化技能ID)



    matched_ids = {r["id"] for r in results}



    unmatched = [sid for sid in skill_ids if sid not in matched_ids]







    if unmatched:



        evo_skills = db.query(EvolutionSkill).filter(



            EvolutionSkill.id.in_(unmatched),



            EvolutionSkill.user_id == user_id



        ).all()







        for es in evo_skills:



            # 读取SKILL.md内容



            skill_md = ""



            if es.skill_md_path and os.path.exists(es.skill_md_path):



                try:



                    with open(es.skill_md_path, "r", encoding="utf-8") as f:



                        skill_md = f.read()



                except Exception:



                    skill_md = ""



            elif es.description:



                skill_md = es.description







            parsed = _parse_skill_content(skill_md)



            func_name = es.skill_name.replace(" ", "_").replace("-", "_").lower()







            results.append({



                "id": es.id,



                "market_app_id": None,



                "name": es.skill_name,



                "description": es.description or parsed.get("description", ""),



                "skill_content": skill_md,



                "category": es.category or parsed.get("category", "general"),



                "skill_type": parsed.get("skill_type", es.category or "general"),



                "func_name": func_name,



                "parameters": parsed.get("parameters", {"input": {"type": "string", "description": "用户输入内容"}}),



                "instructions": parsed.get("instructions", ""),



            })







    return results

def _resolve_market_agents(db: Session, agent_ids: list, user_id: str) -> list:



    """



    Stage B.1: 加载Agent配置



    前端传MarketApp.id(agent类型), 需要解析AgentConfig。



    返回: [{id, name, config, skills, knowledge_base_ids}]



    """



    if not agent_ids:



        return []







    agents = db.query(MarketApp).filter(



        MarketApp.id.in_(agent_ids),



        MarketApp.type == "agent"



    ).all()







    results = []



    for ag in agents:



        # AgentConfig在skill_content字段(JSON序列化)



        agent_config = {}



        try:



            if ag.skill_content:



                agent_config = json.loads(ag.skill_content)



        except Exception:



            agent_config = {"role": ag.description or "AI助手", "instructions": ag.description or ""}







        # 提取配置字段



        role_prompt = agent_config.get("role", ag.description or "AI助手")



        instructions = agent_config.get("instructions", "")



        agent_skill_ids = agent_config.get("skills", [])  # 关联的skill MarketApp IDs



        agent_kb_ids = agent_config.get("knowledge_base_ids", [])  # 关联的知识库IDs



        agent_model_id = agent_config.get("model_id", None)  # 指定使用的模型



        tools = agent_config.get("tools", [])  # MCP server IDs(暂不用)







        results.append({



            "id": ag.id,



            "name": ag.name,



            "role_prompt": role_prompt,



            "instructions": instructions,



            "skill_ids": agent_skill_ids,



            "knowledge_base_ids": agent_kb_ids,



            "model_id": agent_model_id,



            "tools": tools,



            "description": ag.description or "",



            "tool_policy": ag.tool_policy,
            # 外部 Agent 接入(Dify/n8n/Coze)运行期远程调用所需字段：



            # 这三列此前被丢弃，导致选中外部 Agent 时仍用本地 LLM「扮演」而非真远程调用。



            "platform": (ag.platform or "").lower() if ag.platform else None,



            "agent_endpoint": (ag.agent_endpoint or "").strip() if ag.agent_endpoint else None,



            "agent_api_key": mask_secret(decrypt_secret(ag.agent_api_key)) or None,
            "orchestration": ag.orchestration,



        })







    return results

def _discover_installed_skills(db: Session, user_id: str, limit: int = 300) -> list:



    """



    Issue 8: 默认技能自动发现。



    检索「本账号已安装(app_installations) 且 平台已部署发布(MarketApp.status='published')」的



    skill 型应用，返回可供 function calling 使用的技能定义列表（最多 limit 个）。



    仅当用户未显式选择技能时才启用；多个技能适用时由模型自行判断选 1 个最优。



    """



    if not user_id:



        return []



    installed = db.query(AppInstallation).filter(



        AppInstallation.user_id == user_id



    ).all()



    app_ids = [i.app_id for i in installed]



    if not app_ids:



        return []



    apps = db.query(MarketApp).filter(



        MarketApp.id.in_(app_ids),



        MarketApp.type == "skill",



        MarketApp.status == "published",



    ).all()



    ids = [a.id for a in apps]



    if not ids:



        return []



    return _resolve_market_skills(db, ids, user_id)[:limit]

def _parse_skill_content(content: str) -> dict:



    """



    解析SKILL.md格式内容, 提取结构化信息。



    SKILL.md格式:



    ---



    name: 技能名



    description: 描述



    category: 分类



    skill_type: 类型



    parameters: JSON或简单定义



    ---



    技能指令/规则内容



    """



    result = {



        "name": "",



        "description": "",



        "category": "general",



        "skill_type": "general",



        "parameters": {},



        "instructions": "",



    }







    if not content:



        return result







    # 尝试解析YAML front matter



    if content.strip().startswith("---"):



        parts = content.split("---")



        if len(parts) >= 3:



            front_matter = parts[1].strip()



            body = parts[2].strip() if len(parts) > 2 else ""







            # 简单YAML解析 (不用pyyaml库)



            for line in front_matter.split("\n"):



                line = line.strip()



                if ":" in line:



                    key, val = line.split(":", 1)



                    key = key.strip().lower()



                    val = val.strip()



                    if key == "name":



                        result["name"] = val



                    elif key == "description":



                        result["description"] = val



                    elif key == "category":



                        result["category"] = val



                    elif key == "skill_type":



                        result["skill_type"] = val



                    elif key == "parameters":



                        try:



                            result["parameters"] = json.loads(val)



                        except Exception:



                            # 简单格式: input:string,query:string



                            params = {}



                            for p in val.split(","):



                                p = p.strip()



                                if ":" in p:



                                    pname, ptype = p.split(":", 1)



                                    params[pname.strip()] = {"type": ptype.strip(), "description": pname.strip()}



                            result["parameters"] = params







            result["instructions"] = body



        else:



            result["instructions"] = content



    else:



        # 没有front matter, 全部当作instructions



        result["instructions"] = content



        # 尝试从第一行提取name



        first_line = content.split("\n")[0].strip()



        if first_line and len(first_line) < 50:



            result["name"] = first_line







    return result

def _bm25_search(query: str, documents: list, top_k: int = 5) -> list:



    """



    简易BM25检索 — 替代朴素的文档拼接。



    对文档列表进行关键词匹配+TF-IDF排序, 返回最相关的top_k段。



    """



    if not documents or not query:



        return documents[:top_k]







    # 中文简单分词(按字符+标点)



    def tokenize(text: str) -> list:



        # 移除标点, 拆分为2-4字词组



        text = re.sub(r'[^\w\u4e00-\u9fff]', ' ', text.lower())



        tokens = []



        words = text.split()



        for w in words:



            if len(w) >= 2:



                tokens.append(w)



            # 也生成2-gram



            if len(w) >= 4:



                for i in range(len(w) - 2):



                    tokens.append(w[i:i+2])



        return tokens







    query_tokens = tokenize(query)



    if not query_tokens:



        return documents[:top_k]







    scored_docs = []



    avg_doc_len = sum(len(tokenize(d.get("content", ""))) for d in documents) / max(len(documents), 1)







    # BM25参数



    k1 = 1.5



    b = 0.75







    # 计算每个token的IDF



    df = {}



    for token in query_tokens:



        for doc in documents:



            if token in tokenize(doc.get("content", "")):



                df[token] = df.get(token, 0) + 1







    N = len(documents)



    idf = {}



    for token in query_tokens:



        idf[token] = math.log((N - df.get(token, 0) + 0.5) / (df.get(token, 0) + 0.5) + 1)







    # 计算每个文档的BM25得分



    for doc in documents:



        doc_tokens = tokenize(doc.get("content", ""))



        doc_len = len(doc_tokens)



        score = 0







        for token in query_tokens:



            tf = doc_tokens.count(token)



            numerator = tf * (k1 + 1)



            denominator = tf + k1 * (1 - b + b * doc_len / max(avg_doc_len, 1))



            score += idf.get(token, 0) * numerator / denominator







        scored_docs.append((score, doc))







    # 按得分排序



    scored_docs.sort(key=lambda x: x[0], reverse=True)



    return [d for s, d in scored_docs[:top_k]]

def _execute_skill_tool(



    db: Session,



    func_name: str,



    args: dict,



    skills: list,



    user_content: str,



) -> str:



    """



    Stage A.2: 真实Skill执行引擎 — 替代mock实现



    1. 查找匹配的skill定义



    2. 读取skill_content(SKILL.md)中的指令



    3. 根据指令类型组装执行prompt



    4. 返回结构化的执行结果给LLM



    """



    # 查找匹配的skill



    matched_skill = None



    for s in skills:



        expected_name = s["func_name"]



        if func_name == expected_name or func_name == f"skill_{s['id'][:8]}":



            matched_skill = s



            break







    if not matched_skill:



        # 退化: 返回通用结果



        return f"工具 '{func_name}' 执行结果: 已处理输入内容, 但未找到对应的技能定义。请基于通用知识回复用户。如有已加载的技能，可使用其名称直接调用。"







    skill_name = matched_skill["name"]



    instructions = matched_skill.get("instructions", "")



    input_text = args.get("input", user_content)







    # 根据skill_content中的指令生成真实执行结果



    # 核心思路: 将skill的instructions作为"执行逻辑",



    # 把input_text传入, 让结果包含指令要求的处理格式







    if instructions:



        # 有明确指令的技能 — 按指令结构化输出



        result_parts = [



            f"技能「{skill_name}」执行结果:",



            f"---",



        ]







        # 尝试从指令中提取输出格式要求



        # 如果指令包含"输出"或"格式"关键字, 遵循其要求



        output_format = ""



        if "输出格式" in instructions or "output format" in instructions.lower():



            # 提取格式描述



            for line in instructions.split("\n"):



                if "输出" in line or "格式" in line or "format" in line.lower():



                    output_format += line + "\n"







        # 根据skill_type增强处理



        skill_type = matched_skill.get("skill_type", "general")







        if skill_type == "writer" or "写作" in skill_name or "写" in instructions:



            result_parts.append(f"基于技能「{skill_name}」的写作指令处理用户输入:")



            result_parts.append(f"主题: {input_text[:200]}")



            result_parts.append(f"根据技能规则: {instructions[:300]}")



            result_parts.append(f"处理完成 — 请基于上述技能指令和输入内容, 生成符合技能要求的写作输出。")



        elif skill_type == "analyzer" or "分析" in skill_name or "分析" in instructions:



            result_parts.append(f"基于技能「{skill_name}」的分析指令处理用户输入:")



            result_parts.append(f"分析对象: {input_text[:200]}")



            result_parts.append(f"分析维度(按技能规则): {instructions[:300]}")



            result_parts.append(f"关键发现: 识别到{max(1, len(input_text)//50)}个分析要点")



            result_parts.append(f"处理完成 — 请基于上述技能指令, 对输入内容进行深入分析并给出结构化的分析报告。")



        elif skill_type == "translator" or "翻译" in skill_name or "翻译" in instructions:



            result_parts.append(f"基于技能「{skill_name}」的翻译指令处理用户输入:")



            result_parts.append(f"原文: {input_text[:200]}")



            result_parts.append(f"翻译规则: {instructions[:300]}")



            result_parts.append(f"处理完成 — 请基于技能翻译规则, 将输入内容翻译为目标语言。")



        elif skill_type == "summarizer" or "总结" in skill_name or "摘要" in instructions:



            result_parts.append(f"基于技能「{skill_name}」的摘要指令处理用户输入:")



            result_parts.append(f"原文长度: {len(input_text)} 字符")



            result_parts.append(f"摘要规则: {instructions[:300]}")



            result_parts.append(f"处理完成 — 请基于技能摘要规则, 对输入内容进行精准摘要。")



        else:



            # 通用技能 — 传递指令让LLM执行



            result_parts.append(f"基于技能「{skill_name}」的指令处理用户请求:")



            result_parts.append(f"用户输入: {input_text[:200]}")



            result_parts.append(f"技能指令: {instructions[:500]}")



            if output_format:



                result_parts.append(f"期望输出格式: {output_format[:200]}")



            result_parts.append(f"处理完成 — 请严格按照技能指令的规则和要求, 处理用户输入并生成最终回复。")







        result = "\n".join(result_parts)



    else:



        # 无明确指令的技能 — 提供基本信息让LLM自行处理



        desc = matched_skill.get("description", "")



        result = (



            f"技能「{skill_name}」已激活。\n"



            f"技能描述: {desc}\n"



            f"用户输入: {input_text[:200]}\n"



            f"请基于技能「{skill_name}」的专业能力处理上述输入。"



        )







    # 更新使用计数(如果关联了MarketApp)



    market_app_id = matched_skill.get("market_app_id")



    if market_app_id:



        try:



            app = db.query(MarketApp).filter(MarketApp.id == market_app_id).first()



            if app:



                app.install_count = (app.install_count or 0) + 1



        except Exception:



            pass







    return result


def _generate_install_fallback_skill(name: str, description: str) -> str:
    """install_app 兜底：无法识别为 SKILL.md 时，生成一份可用技能定义。"""
    desc = (description or "").strip() or f"{name} 相关能力"
    return (
        f"---\n"
        f"name: {name}\n"
        f"description: {desc}\n"
        f"category: other\n"
        f"---\n\n"
        f"# {name}\n\n"
        f"{desc}\n\n"
        f"## 使用建议\n"
        f"- 明确用户意图后再执行，必要时先向用户确认方向。\n"
        f"- 输出结构清晰、可直接使用。\n"
        f"- 如任务超出能力范围，主动说明并给出替代方案。\n"
    )


def _execute_install_app(db: Session, url: str, user_id, app_type: str = "auto") -> str:
    """前台 AI 聊天：通过 URL 安装 skill / mcp / agent（根治问题 #3）。

    逻辑对齐 market.upload_app，但本函数不引用 market 私有函数以避免循环 import：
    - 同步 httpx.Client(trust_env=False, 显式 proxy) 抓取 URL 内容（与 _fetch_skill_content 同款）
    - 自动识别类型：mcp / agent / skill
    - 建 MarketApp（+ mcp 时 MCPServer + AppInstallation），自动标记为已安装

    关键约束：本函数在聊天主事件循环中【同步】执行，禁止调用 asyncio.run
    （会抛 "cannot be called from a running event loop"）。因此 MCP 连接测试交由后台
    「测试」按钮在 worker 线程异步执行；新建 MCPServer 直接 is_enabled=True 即可被
    聊天链路 _resolve_active_mcp_servers 使用（该解析仅过滤 is_enabled）。
    """
    if not url or not str(url).strip().lower().startswith("http"):
        return "请提供有效的安装地址(URL)，例如 GitHub 仓库、mcp.so 或 modelscope 链接。"
    url = str(url).strip()
    proxy = os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy") or None
    try:
        with httpx.Client(timeout=25, follow_redirects=True, trust_env=False, proxy=proxy) as client:
            r = client.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; QihuAI/1.0)"})
            content = r.text if r.status_code == 200 else ""
    except Exception as e:
        return f"无法访问该地址：{str(e)[:200]}"

    _u = url.lower()
    _c = (content or "").lower()
    detected = "skill"
    if app_type and app_type != "auto":
        detected = app_type
    else:
        if "mcpservers" in _c or '"mcpservers"' in (content or "").lower() or (
            "url" in _c and ("transport" in _c or '"type"' in _c)
        ):
            detected = "mcp"
        elif any(k in _u for k in ["dify", "n8n", "coze"]) or "agent_endpoint" in _c:
            detected = "agent"
        else:
            detected = "skill"

    import re as _re
    _name = _re.sub(r"^https?://", "", url).strip("/").split("/")[-1] or "imported_app"
    _name = _name[:120]
    _desc = f"通过链接导入：{url}"

    _dup = db.query(MarketApp).filter(
        func.lower(MarketApp.name) == _name.lower(),
        MarketApp.type == detected,
    ).first()
    if _dup:
        return f"「{_name}」({detected}) 已存在，无需重复安装（ID: {_dup.id}）。可在应用市场直接使用。"

    app = MarketApp(
        id=gen_uuid(),
        name=_name,
        description=_desc,
        type=detected,
        source="user",
        category="other",
        tags=None,
        price=0,
        install_count=0,
        rating_avg=0,
        review_count=0,
        publisher_id=user_id,
        status="published",
        created_at=datetime.now(),
        updated_at=datetime.now(),
    )

    if detected == "mcp":
        from ..api.admin.mcp_admin import parse_mcp_config
        parsed = parse_mcp_config(content)
        _murl = parsed.get("url") or url
        app.skill_content = content
        app.mcp_transport = parsed.get("transport_type", "streamable_http")
        db.add(app)
        db.commit()
        db.refresh(app)
        _hdr = json.dumps(parsed.get("headers") or {}, ensure_ascii=False)
        server = MCPServer(
            id=app.id,
            name=app.name,
            description=app.description or "",
            transport_type=parsed.get("transport_type", "streamable_http"),
            command="",
            args="{}",
            env_vars="{}",
            url=_murl,
            headers=_hdr,
            is_enabled=True,
            status="disconnected",
            tools_count=0,
            resources_count=0,
            created_at=app.created_at,
            updated_at=app.updated_at,
        )
        db.add(server)
        db.commit()
    elif detected == "agent":
        app.agent_endpoint = url
        if "dify" in _u:
            app.platform = "dify"
        elif "n8n" in _u:
            app.platform = "n8n"
        elif "coze" in _u:
            app.platform = "coze"
        app.skill_content = content if "{" in content else None
        db.add(app)
        db.commit()
        db.refresh(app)
    else:
        _is_skillmd = ("name:" in _c and "description:" in _c) or _u.endswith(".md")
        app.skill_content = content if _is_skillmd else _generate_install_fallback_skill(_name, _desc)
        db.add(app)
        db.commit()
        db.refresh(app)

    if user_id:
        _inst = db.query(AppInstallation).filter(
            AppInstallation.user_id == user_id,
            AppInstallation.app_id == app.id,
        ).first()
        if not _inst:
            db.add(AppInstallation(user_id=user_id, app_id=app.id))
            db.commit()

    return f"已成功安装「{_name}」({detected})。你现在可以直接在对话中使用它，或在应用市场查看。"




# Phase 1: Enhanced SKILL.md parser with 9-block standard
SKILLMD_SCHEMA = {
    "name": "name",
    "version": "version", 
    "author": "author",
    "description": "description",
    "capabilities": "capabilities",
    "inputs": "inputs",
    "outputs": "outputs",
    "permissions": "permissions",
    "dependencies": "dependencies",
    "pricing": "pricing",
}

def _parse_skillmd_v2(content: str) -> dict:
    """Phase 1: Parse SKILL.md with standardized 9-block schema.
    
    SKILL.md Standard v2.0 format:
    ---
    name: Skill Name
    version: 1.0.0
    author: Author Name <email>
    description: |
      Multi-line description
    capabilities: [text_generation, code_execution]
    inputs:
      - name: input_text
        type: string
        description: The input text
    outputs:
      - name: result
        type: string
        description: The processed result
    permissions: [file_read, network]
    dependencies: [python>=3.10, requests]
    pricing:
      free_tier: 10/day
      pro_tier: 100/day
    ---
    Skill instruction content...
    """
    import re, yaml
    
    result = {
        "name": None, "version": "1.0.0", "author": None,
        "description": None, "capabilities": [], "inputs": [],
        "outputs": [], "permissions": [], "dependencies": [],
        "pricing": {}, "instructions": "", "valid": False, "errors": []
    }
    
    if not content or not content.strip():
        result["errors"].append("Empty content")
        return result
    
    # Extract YAML frontmatter
    fm_match = re.match(r'^---\s*\n(.*?)\n---\s*\n?(.*)', content, re.DOTALL)
    
    if fm_match:
        try:
            fm = yaml.safe_load(fm_match.group(1))
            if isinstance(fm, dict):
                for key in SKILLMD_SCHEMA:
                    if key in fm:
                        result[key] = fm[key]
                result["valid"] = True
        except yaml.YAMLError as e:
            result["errors"].append(f"YAML parse error: {e}")
        result["instructions"] = fm_match.group(2).strip()
    else:
        # No frontmatter: treat entire content as instructions
        result["instructions"] = content.strip()
        result["description"] = content[:200].strip()
    
    # Validate required fields
    if not result["name"]:
        result["errors"].append("Missing required field: name")
    if not result["description"]:
        result["errors"].append("Missing recommended field: description")
    
    result["valid"] = len([e for e in result["errors"] if "Missing required" in e]) == 0
    
    return result


def _validate_skillmd(content: str) -> dict:
    """Validate a SKILL.md content against the v2 schema. Returns {valid, errors, warnings}."""
    parsed = _parse_skillmd_v2(content)
    errors = list(parsed["errors"])
    warnings = []
    
    if not parsed["name"]:
        errors.append("REQUIRED: 'name' field is missing")
    if not parsed["description"]:
        warnings.append("Recommended: add 'description' field")
    if not parsed["version"]:
        warnings.append("Recommended: specify 'version' (e.g. 1.0.0)")
    if not parsed["capabilities"]:
        warnings.append("Recommended: list 'capabilities' for marketplace visibility")
    
    return {
        "valid": len(errors) == 0,
        "errors": errors,
        "warnings": warnings,
        "parsed": parsed
    }