Files
qihuaiflow/backend/python/app/api/im_llm.py
T

520 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
IM Bot 真实 LLM 调用。
原实现位于 channels.py(通用频道管理模块),该模块已移除。
为保持 IM Bot(微信机器人等)正常生成回复,将 _call_real_llm 迁移至此独立模块。
行为(对齐 AI 聊天 — 2026-07-23 同步至最新):
- Issue 7:图像/视频/语音意图优先走平台媒体模型(与 AI 聊天同一套 Auto 路由),
不再对"制作图片"这类请求只回文本。
- Issue 8:默认 Auto 模型;每轮先自动发现本账号已安装且平台已部署的技能,
多个适用时由模型自行判断选 1 个最优,通过 function calling 执行。
- 2026-07-15:修复 trust_env=False + proxy 处理,避免容器坏代理导致调用失败;
媒体生成前先经 _optimize_media_prompt 精炼提示词,提升 AGNES 生成质量。
- 2026-07-23:对齐 Web Chat 最新能力:
* 模型 failover(Auto 模式自动切换备选模型)
* 平台知识库注入(get_platform_knowledge,技能清单实时同步)
"""
import asyncio
import os
import json
import logging
import re as _re
from datetime import datetime, timezone, timedelta
from typing import Optional
from urllib.parse import urlparse as _urlparse
import httpx
from app.core.llm_client import chat_complete # 统一模型层(07-24): web/IM 共用标准 OpenAI SDK 客户端
from sqlalchemy.orm import Session
from ..capability_registry import resolve_auto_model
from ..models import AIModel, ModelProvider, TokenUsageLog, gen_uuid
from app.core.observability import get_trace_id
# #875 LLM Router:统一模型路由层(影子模式先行,收敛 litellm 主密钥硬编码)
from ..core.llm_router import (
route as _llm_route,
set_route_ctx as _set_route_ctx,
SHADOW as _LLM_ROUTER_SHADOW,
resolve_api_key as _llm_resolve_api_key,
)
logger = logging.getLogger(__name__)
# 香港出网代理(实测对 AGNES 延迟更低,2026-07-15 用户实测),与 chat.py 同值。
_HK_PROXY = "http://qyyfw:XHR521yhf@103.242.14.208:11495"
def _proxy_for_url(url: str) -> str | None:
"""按目标域名选择是否走代理(与 chat.py 同逻辑)。
2026-07-15 调整:AGNES 改走香港代理(实测延迟更低),不再直连。
国内域名(尊重 NO_PROXY)直连,更稳定。
"""
import os
host = (_urlparse(url).hostname or "").lower()
if "agnes-ai.com" in host or "agnes-ai.space" in host:
return _HK_PROXY
# 2026-07-30 修复 3+: 不代理 Docker 内部 hostname 和 localhost
if "qihuyun.com" in host or "localhost" in host or host.startswith("127.") or "qihu-aiflow" in host:
return None
_no_proxy = [h.strip().lower() for h in os.environ.get("NO_PROXY", "").split(",") if h.strip()]
for _np in _no_proxy:
if host == _np or host.endswith("." + _np) or _np in host:
return None
return os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or "http://172.18.0.1:7890"
_DEBUG_LOG_PATH = "/tmp/im_llm_debug.log"
def _debug_log(msg: str):
"""写磁盘调试日志(不依赖 uvicorn logging 配置)。"""
try:
with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as _f:
_f.write(datetime.now(timezone(timedelta(hours=8))).isoformat() + " " + msg + "\n")
except Exception:
pass
def _run_async(coro, timeout=None):
"""在同步上下文中运行协程,兼容「无事件循环」与「已有事件循环」两种场景。
IM 个人微信在独立线程中调用(无事件循环)→ asyncio.run;
IM 多平台 Webhook 可能在 FastAPI 协程内调用(已有事件循环)→ 丢到线程池执行。
timeout: 可选,秒数;超时抛出 asyncio.TimeoutError(由调用方捕获)。
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if timeout is not None:
async def _timed():
return await asyncio.wait_for(coro, timeout=timeout)
target = _timed()
else:
target = coro
if loop is not None and loop.is_running():
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(1) as ex:
return ex.submit(lambda: asyncio.run(target)).result(timeout=(timeout + 10) if timeout else None)
return asyncio.run(target)
def _log_media_usage(db: Session, user_id: str, ai_model, media_type: str, prompt: str):
"""IM Bot 媒体生成记账:把图片/视频消耗写入 TokenUsageLog,供驾驶舱统计。
此前 IM Bot 路径完全不写 TokenUsageLog,导致微信生成的图片/视频在驾驶舱里"查无此项"。
媒体 API 不返回 token 用量,这里用与 chat.py _generate_media_task 一致的估算。
"""
if not user_id or not ai_model:
return
try:
base = 1200 if media_type == "video" else 400
est = base + (len(prompt or "") // 4)
db.add(TokenUsageLog(
id=gen_uuid(),
user_id=user_id,
model_id=ai_model.id,
model_name=ai_model.name,
trace_id=get_trace_id(),
prompt_tokens=est,
completion_tokens=0,
total_tokens=est,
created_at=datetime.now(timezone(timedelta(hours=8))).replace(tzinfo=None),
))
db.commit()
except Exception as e:
try:
db.rollback()
except Exception:
pass
logger.warning(f"[im_llm] 媒体 Token 记账失败(忽略): {e}")
def _call_real_llm(db: Session, user=None, prompt: str = "", platform: str = "im", user_id: str = None) -> "str | dict":
"""调用七虎AI 生成回复,供 IM Bot 复用。支持媒体意图路由、已安装技能自动发现、模型 failover。"""
try:
from app.api.chat import (
_generate_media,
detect_media_intent,
_discover_installed_skills,
_build_system_prompt,
_execute_skill_tool,
_list_enabled_mcp_servers,
)
# 2026-08-04 双 Claw 对齐:能力发现收敛为唯一实现,Web / IM 共用
from app.core.capability_discovery import (
build_mcp_tool_defs,
discover_mcp_servers,
discover_skills,
)
from app.mcp_client import call_mcp_tool
# ===== 1) 媒体意图路由(Issue 7)=====
# 2026-07-30 fix: 只对当前用户消息做媒体意图检测,避免历史中的图像请求污染
_cur_msg = prompt
_marker = "用户当前问题:"
if _marker in prompt:
_cur_msg = prompt[prompt.rfind(_marker) + len(_marker):].strip()
intent = detect_media_intent(_cur_msg)
if intent:
ai_model = resolve_auto_model(intent, db)
if ai_model:
try:
result = _run_async(_generate_media(db, ai_model, prompt, user_id), timeout=300)
except Exception:
# 任何未预期异常(如 AGNES 连接重置)统一兜底为友好提示,
# 绝不直接把原始异常(如 "Server disconnected")甩给用户
result = "⚠️ 媒体生成暂时失败(连接或服务波动),请稍后重试。"
if result:
# 媒体生成成功:返回结构化对象,交由 im_bot_user 在微信内联发送图片/视频
s = result.strip()
if s.startswith("<img") or s.startswith("<video"):
m = _re.search(r'src="([^"]+)"', s)
url = m.group(1) if m else ""
media_type = "image" if s.startswith("<img") else "video"
if not url:
return "(生成完成,但未能提取媒体地址)"
# IM Bot 路径此前完全不记账 → 补写媒体消耗,驾驶舱才看得到图片/视频数据
_log_media_usage(db, user_id, ai_model, media_type, prompt)
return {"__media__": True, "media_type": media_type, "url": url}
# 媒体生成失败:_generate_media 已返回友好中文(如限频/队列繁忙),
# 直接透传,绝不把原始提示词/标签/JSON 透传给用户
if s.startswith("⚠️") or s.startswith("(媒体"):
return s
return "(媒体生成未能返回有效内容,请重试)"
# ===== 1.5) EngineCore 薄壳(#873 · 灰阶门,默认关闭)=====
# IM_USE_ENGINE_CORE=1 时,文本回复改走与 Web Claw 同一份 EngineCore
# (_send_message_core),经 BufferAdapter 缓冲装配为整条回复;上方媒体快路径
# 已提前返回,不受此门影响。默认 0/未设置 = 走下方既有自包含逻辑(零改动)。
if os.environ.get("IM_USE_ENGINE_CORE") == "1":
return _run_async(
_call_engine_core(db, user_id, prompt, platform, user),
timeout=300,
)
# ===== 2) 文本回复:模型 failover + 技能自动发现 + 平台知识库注入 =====
from ..platform_knowledge import get_platform_knowledge
# 构建候选模型列表(对齐 Web Chat failover)
ai_model = resolve_auto_model("text", db)
_try_models = [ai_model] if (ai_model and ai_model.is_enabled) else []
# Auto 模式附加备选(IM Bot 始终 Auto)
# 2026-07-31 根因修复:仅选用「纯文本」模型作为 failover 备选。
# 此前按 is_enabled 顺序取前 4 个,会混入图像/视频/语音(ASR)模型
# (这些模型 capability 含 "text" 但本质是媒体模型,收到文本 prompt 返回空内容、
# 不抛异常 → 被误判为「模型失败」,最终导致「已尝试N个模型均调用失败」)。
_MEDIA_CAPS = {"image", "video", "audio", "asr", "embedding", "rerank", "ocr"}
def _is_text_only(m):
_caps = m.capabilities or []
return "text" in _caps and not any(c in _MEDIA_CAPS for c in _caps)
_fallbacks = db.query(AIModel).filter(
AIModel.is_enabled == True,
AIModel.id != (ai_model.id if ai_model else ""),
).order_by(AIModel.is_default.desc()).all()
_fallbacks = [m for m in _fallbacks if _is_text_only(m)]
_try_models.extend([m for m in _fallbacks[:4] if m not in _try_models])
# ===== LLM Router(#875 影子模式)=====
# 计算决策并打 claw_type 标签 metrics;影子模式下仅观测,不改流量。
# 切主路径:LLM_ROUTER_SHADOW=0 时改用 Router 候选链(已含文本过滤,等同此处逻辑)。
try:
_set_route_ctx(tenant_id=getattr(user, "tenant_id", "") or "", claw_type="im_wechat")
_rm = _llm_route("text", db=db, legacy_primary=_try_models[0].name if _try_models else None)
if not _LLM_ROUTER_SHADOW and _rm.models:
_try_models = _rm.models
_debug_log(f"ROUTER_LIVE models={[m.name for m in _try_models]}")
except Exception as _e:
logger.warning(f"[im_llm] LLM Router 观测失败(降级忽略): {_e}")
_debug_log(f"TRY_MODELS user={user_id} count={len(_try_models)} names={[m.name for m in _try_models]}")
if not _try_models:
return "(未配置可用文本模型,无法生成回复)"
# 能力自动发现(提前执行一次,不必每个候选模型重复做)
# 2026-08-04 根因修复:
# 旧实现 `_discover_installed_skills(db, user_id)[:15]` 强依赖
# app_installations 表,线上该表 0 行 → 97 个 published skill
# 在 Web / IM 双端同时发现不到;`[:15]` 截断的是一个空列表。
# 现改走统一发现层(安装优先 → published 降级 → 相关性裁剪)。
_query_hint = _cur_msg or prompt
skills = []
try:
skills = discover_skills(db, user_id, query=_query_hint, claw="im")
except Exception as _e:
logger.warning(f"[im_llm] 技能发现失败(降级为空): {_e}")
skills = []
# MCP 自动挂载:IM Bot Claw 没有勾选 UI,自动发现即其等价通道。
# (Web Claw 靠用户在界面勾选 → effective_mcp_ids)
mcp_servers = []
mcp_function_definitions = []
mcp_tool_map = {}
try:
mcp_servers = discover_mcp_servers(db, None, claw="im")
if mcp_servers:
mcp_function_definitions, mcp_tool_map = _run_async(
build_mcp_tool_defs(mcp_servers), timeout=45
)
except Exception as _e:
logger.warning(f"[im_llm] MCP 发现失败(降级为空): {_e}")
mcp_servers, mcp_function_definitions, mcp_tool_map = [], [], {}
_debug_log(
f"CAP skills={len(skills)} mcp_servers={len(mcp_servers)} "
f"mcp_tools={len(mcp_function_definitions)}"
)
# 平台知识库注入(对齐 Web Chat——微信用户提问"平台有哪些技能"也能得到完整清单)
kb_context = ""
try:
pk = get_platform_knowledge(db=db)
if pk:
kb_context = f"【平台知识库——优先使用以下知识回答用户问题】\n{pk}"
except Exception:
pass
# 尝试每个候选模型
_retry_log = []
for _try_model in _try_models:
ai_model = _try_model
provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first()
api_key = ai_model.api_key or (provider.api_key if provider else None)
api_url = ai_model.endpoint
# 2026-07-30: LiteLLM 端点统一用 master key(收敛进 LLM Router,两端唯一来源)
api_key = _llm_resolve_api_key(api_url, api_key, provider.api_key if provider else None)
if not api_url and provider and provider.api_base_url:
api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions"
if not (api_key and api_url):
_retry_log.append(f"{ai_model.name}: 缺少密钥或端点")
continue
# agents 暂留空:线上 market_apps 中 type='agent' 记录为 0,
# 且 IM 侧尚无 agent 执行分支;待薄壳化接入 EngineCore 后自动获得。
system_prompt, function_definitions = _build_system_prompt(
skills=skills,
agents=[],
kb_context=kb_context,
user_content=prompt,
mcp=mcp_servers,
mcp_functions=mcp_function_definitions,
auto_discovered=bool(skills),
all_mcp=_list_enabled_mcp_servers(db),
db=db,
)
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt},
]
# 尝试调用
try:
max_rounds = 3
ai_content = ""
_http_ok = True
for round_num in range(max_rounds):
request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048, "enable_thinking": False, "temperature": 0.7}
if function_definitions:
request_body["tools"] = [
{"type": "function", "function": fd} for fd in function_definitions
]
request_body["tool_choice"] = "auto"
try:
import logging as _l5
_l5.warning(f'[im_llm REQ] {ai_model.name}: url={api_url[:60]}, proxy={_proxy_for_url(api_url)}, api_key={api_key[:20]}..., messages_count={len(request_body.get("messages", []))}, tools_count={len(request_body.get("tools", []))}')
_debug_log(f"REQ model={ai_model.name} url={api_url} proxy={_proxy_for_url(api_url)} body={json.dumps(request_body, ensure_ascii=False)[:600]}")
data = chat_complete(
api_url,
api_key,
request_body,
proxy=_proxy_for_url(api_url),
timeout=ai_model.timeout or 120,
)
_l5.warning(f'[im_llm RES] {ai_model.name}: data_keys={list(data.keys()) if data else None}, resp_type={type(data).__name__}')
_debug_log(f"RES model={ai_model.name} data={json.dumps(data, ensure_ascii=False)[:600]}")
except Exception as _exc:
_retry_log.append(f"{ai_model.name}: {str(_exc)[:60]}")
import traceback as _tb
logger.warning(f"[im_llm] 模型 {ai_model.name} 异常: {_exc}")
logger.warning(f"[im_llm] {ai_model.name} 异常详情: {_tb.format_exc()[:2000]}")
_debug_log(f"EXC model={ai_model.name} err={str(_exc)[:200]} trace={_tb.format_exc()[:500]}")
_http_ok = False
break
message = data.get("choices", [{}])[0].get("message", {})
tool_calls = message.get("tool_calls", [])
# 2026-07-30 CRITICAL DEBUG
import logging as _lcr
_raw_content = message.get("content")
_raw_reasoning = message.get("reasoning_content")
_msg_keys = list(message.keys()) if message else []
_tool_calls = tool_calls
_lcr.warning(f'[CRITICAL] MODEL={ai_model.name} msg_keys={_msg_keys} content_len={len(_raw_content or "")} reasoning_len={len(_raw_reasoning or "")} has_tool_calls={bool(_tool_calls)}')
_debug_log(f"PARSE model={ai_model.name} content_len={len(_raw_content or '')} reasoning_len={len(_raw_reasoning or '')} content_preview={repr(_raw_content)[:120]}")
ai_content = message.get("content", "") or message.get("reasoning_content") or ""
if not ai_content and (message.get("reasoning_content") or ""):
ai_content = f"(AI 思考:{message.get('reasoning_content')[:500]}...)"
if tool_calls and (skills or mcp_tool_map):
messages.append({
"role": "assistant",
"content": message.get("content") or message.get("reasoning_content") or "",
"tool_calls": tool_calls,
})
for tc in tool_calls:
func_name = tc.get("function", {}).get("name", "")
try:
func_args = json.loads(tc.get("function", {}).get("arguments", "{}"))
except Exception:
func_args = {"input": tc.get("function", {}).get("arguments", "")}
if func_name.startswith("mcp__"):
# MCP 工具:命名规则与 Web Claw 完全一致
# mcp__{server}__{tool}__{idx}
_info = mcp_tool_map.get(func_name)
if not _info:
tool_result = (
"[MCP 工具未找到] 可用工具: "
+ str(list(mcp_tool_map.keys())[:5])
)
else:
_srv = next(
(s for s in mcp_servers
if s.get("id") == _info.get("server_id")),
None,
)
if not _srv:
tool_result = f"[MCP 服务未找到: {_info.get('server_name')}]"
else:
try:
tool_result = _run_async(
call_mcp_tool(_srv, _info["name"], func_args),
timeout=90,
)
except Exception as _mexc:
tool_result = (
f"[MCP 工具 {_info.get('name')} 执行失败:"
f"{str(_mexc)[:160]}]"
)
_debug_log(
f"MCP_CALL name={func_name} "
f"result_len={len(tool_result or '')}"
)
else:
tool_result = _execute_skill_tool(db, func_name, func_args, skills, prompt)
messages.append({
"role": "tool",
"tool_call_id": tc.get("id", "call_1"),
"content": tool_result,
})
if round_num >= max_rounds - 1:
break
continue
break
# 检查本轮调用是否成功
import logging as _log2
_log2.warning(f'[im_llm trace] {ai_model.name}: http_ok={_http_ok}, ai_content_len={len(ai_content) if ai_content else 0}, ai_content_preview={repr(ai_content)[:200]}')
_debug_log(f"CHECK model={ai_model.name} http_ok={_http_ok} ai_content_len={len(ai_content) if ai_content else 0}")
if _http_ok and ai_content:
_debug_log(f"SUCCESS model={ai_model.name} returning len={len(ai_content)}")
return ai_content or "(AI 回复为空)"
# 本轮失败:记录日志,切换备选
logger.warning('[im_llm] 模型 %s 失败 (http_ok=%s, content=%s), 切换备选...' % (
ai_model.name, _http_ok, repr(ai_content)[:100]
))
_debug_log(f"FAIL model={ai_model.name} http_ok={_http_ok} content_preview={repr(ai_content)[:100]}")
continue
except Exception as exc:
_retry_log.append(f"{ai_model.name}: {str(exc)[:60]}")
logger.warning(f"[im_llm] 模型 {ai_model.name} 异常: {exc}")
_debug_log(f"OUTER_EXC model={ai_model.name} err={str(exc)[:120]}")
continue
# 全部失败
_summary = "; ".join(_retry_log[-3:])
_debug_log(f"ALL_FAILED try_models={len(_try_models)} retry_log={_retry_log}")
return f"(已尝试 {len(_try_models)} 个模型均调用失败。最后错误: {_summary}。请稍后重试。)"
except Exception as e:
return f"(AI 回复生成失败:{e})"
async def _call_engine_core(db, user_id, prompt: str, platform: str, user=None) -> "str | dict":
"""(#873 薄壳)复用 Web Claw 同一份 EngineCore 生成 IM 回复。
单引擎原则:IM Bot 不再维护第二份「模型 failover + 技能/MCP 执行」实现,
而是把已发现的 skill_ids/mcp_ids 交给 _send_message_core,经 BufferAdapter
缓冲装配成整条回复(媒体结构 __media__ 由 BufferAdapter 原样保留)。
默认不启用(见 _call_real_llm 中的 IM_USE_ENGINE_CORE 灰阶门)。
"""
from app.api.chat_service import _send_message_core
from app.api.chat_utils import SendMessageRequest
from app.core.claw_adapters import BufferAdapter
from ..models import Conversation, User
# 解析用户:IM 调用方常传 user=None,仅给 user_id
real_user = user
if real_user is None and user_id:
real_user = db.query(User).filter(User.id == user_id).first()
if real_user is None:
return "(EngineCore 模式:无法解析当前用户,请检查 IM 账号绑定)"
# 能力发现(与 IM 自包含逻辑同一套,claw="im")
from app.core.capability_discovery import (
discover_mcp_servers,
discover_skills,
)
_query_hint = prompt
_marker = "用户当前问题:"
if _marker in prompt:
_query_hint = prompt[prompt.rfind(_marker) + len(_marker):].strip()
skills = []
try:
skills = discover_skills(db, user_id, query=_query_hint, claw="im") or []
except Exception as _e:
logger.warning(f"[im_llm.engine] 技能发现失败(降级为空): {_e}")
mcp_servers = []
try:
mcp_servers = discover_mcp_servers(db, None, claw="im") or []
except Exception as _e:
logger.warning(f"[im_llm.engine] MCP 发现失败(降级为空): {_e}")
skill_ids = [
(s.get("id") if isinstance(s, dict) else getattr(s, "id", None))
for s in skills
]
skill_ids = [s for s in skill_ids if s]
mcp_ids = [
(m.get("id") if isinstance(m, dict) else getattr(m, "id", None))
for m in mcp_servers
]
mcp_ids = [m for m in mcp_ids if m]
# 构造本回合专用对话:EngineCore 要求已存在且归属当前用户的 Conversation
conv = Conversation(
id=gen_uuid(),
user_id=real_user.id,
title=f"IM对话-{platform}",
source="im_bot",
)
db.add(conv)
db.commit()
req = SendMessageRequest(
content=prompt,
skill_ids=skill_ids or None,
mcp_ids=mcp_ids or None,
use_tools=bool(skill_ids or mcp_ids),
agent_ids=None,
knowledge_base_ids=None,
attachments=None,
)
adapter = BufferAdapter(tenant_id=str(getattr(real_user, "tenant_id", "") or ""))
await _send_message_core(conv.id, req, db, real_user, stream=True, on_event=adapter)
return adapter.assemble()