fix: 双Claw能力对齐 + MCP/LLM代理绕过（P0止血）
- 新增 app/core/capability_discovery.py：统一能力发现层，Web/IM 双Claw共用单一实现 （三级降级：用户安装 → 平台published → 相关性裁剪；MCP 自动挂载已启用服务） - im_llm.py：IM Bot Claw 改用统一发现层，恢复 skills(12) + MCP(26工具) 能力， 修复 #866 部署回退导致的 agents=[]/mcp=[] 回归；mcp__ 前缀分流到 call_mcp_tool - mcp_client.py：注入 httpx_client_factory(trust_env=False) 绕过容器 HTTP_PROXY， 修复外部 MCP(xiaobenyang.com / anysearch.com) tools/list 全部静默失败（mcp_tools 0→26） - llm_client.py / im_bot_user.py：2026-07-31 代理劫持内部请求 + 媒体断连修复（随镜像已上线）
This commit is contained in:
QihuAI Design committed
1 parent
9a43ee244d
commit
29a1ce2e54
5 files changed
+575
-55
No files matched your search
@@ -0,0 +1,277 @@
|
||||
"""统一能力发现层（双 Claw 共用 · 单一实现）。
|
||||
|
||||
背景
|
||||
----
|
||||
七虎平台对外有两个 Claw 端口：
|
||||
* Web Claw —— SSE 流式，用户可在 UI 勾选 skills / mcp / agents
|
||||
* IM Bot Claw —— 微信等 IM，server push 整条回复，**没有任何勾选 UI**
|
||||
|
||||
历史问题（2026-08-04 线上实证）：
|
||||
1. `im_llm.py` 写死 `agents=[]` / `mcp=[]`，IM 端永远拿不到 MCP 工具；
|
||||
2. `_discover_installed_skills()` 强依赖 `app_installations` 表，而线上该表
|
||||
**0 行** → 97 个 published skill 在 Web / IM **双端同时发现不到**；
|
||||
3. IM 端 skills 还被 `[:15]` 截断（截断一个空列表，属噪声）。
|
||||
|
||||
设计原则
|
||||
--------
|
||||
能力发现只允许有 **一份实现**（本模块）。任何 Claw 想要"技能/工具"，
|
||||
都必须调用这里的函数；禁止在端侧另写一套发现逻辑——历史上 #866 部署
|
||||
就是因为存在第二份实现而把 #764/#765 的注入改动整体覆盖回退。
|
||||
|
||||
发现策略（三级降级，双端一致）
|
||||
--------------------------
|
||||
1) 用户显式安装（app_installations）—— 最高优先，尊重用户意愿
|
||||
2) 无安装记录 → 降级为平台已发布（status='published'）全集
|
||||
3) 池子超过预算 → 按当前问题做轻量相关性排序，取 top-K 控制 token
|
||||
|
||||
运维旋钮（环境变量，改完重启生效）
|
||||
--------------------------------
|
||||
CAP_SKILL_LIMIT 单轮注入技能数上限，默认 12
|
||||
CAP_MCP_LIMIT 单轮注入 MCP 服务数上限，默认 8
|
||||
CAP_SKILL_FALLBACK_PUBLISHED 安装表为空时是否降级为 published，默认 开
|
||||
CAP_MCP_AUTO 未显式指定时是否自动挂载已启用 MCP，默认 开
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Iterable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"discover_skills",
|
||||
"discover_mcp_servers",
|
||||
"build_mcp_tool_defs",
|
||||
"rank_by_query",
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 环境旋钮
|
||||
# --------------------------------------------------------------------------
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
try:
|
||||
raw = (os.environ.get(name) or "").strip()
|
||||
return int(raw) if raw else default
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _env_flag(name: str, default: bool = True) -> bool:
|
||||
raw = (os.environ.get(name) or "").strip().lower()
|
||||
if not raw:
|
||||
return default
|
||||
return raw in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 轻量相关性排序（中文友好：整词 + 2-gram）
|
||||
# --------------------------------------------------------------------------
|
||||
def _tokens(text: str) -> set:
|
||||
text = re.sub(r"[^\w\u4e00-\u9fff]+", " ", (text or "").lower())
|
||||
out: set = set()
|
||||
for w in text.split():
|
||||
if len(w) >= 2:
|
||||
out.add(w)
|
||||
for i in range(len(w) - 1):
|
||||
out.add(w[i : i + 2])
|
||||
return out
|
||||
|
||||
|
||||
def rank_by_query(items: list, query: str, top_k: int) -> list:
|
||||
"""按与 query 的词元重叠度排序取前 top_k。
|
||||
|
||||
池子本来就不超预算时原样返回，避免无谓抖动（同一问题两次调用结果稳定）。
|
||||
"""
|
||||
if not items:
|
||||
return []
|
||||
if top_k <= 0 or len(items) <= top_k:
|
||||
return items[:top_k] if top_k > 0 else []
|
||||
q = _tokens(query)
|
||||
if not q:
|
||||
return items[:top_k]
|
||||
|
||||
scored = []
|
||||
for idx, it in enumerate(items):
|
||||
blob = " ".join(
|
||||
str(it.get(k) or "")
|
||||
for k in ("name", "description", "category", "func_name")
|
||||
)
|
||||
t = _tokens(blob)
|
||||
if t:
|
||||
# 交集数按文档长度做平方根归一，抑制"长描述天然占优"
|
||||
score = len(q & t) / (len(t) ** 0.5)
|
||||
else:
|
||||
score = 0.0
|
||||
# idx 作为稳定次序的 tie-breaker，保证同分时结果可复现
|
||||
scored.append((-score, idx, it))
|
||||
scored.sort(key=lambda x: (x[0], x[1]))
|
||||
return [it for _, _, it in scored[:top_k]]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 技能发现
|
||||
# --------------------------------------------------------------------------
|
||||
def discover_skills(
|
||||
db,
|
||||
user_id: str | None,
|
||||
query: str = "",
|
||||
limit: int | None = None,
|
||||
claw: str = "web",
|
||||
) -> list:
|
||||
"""发现当前请求可用的技能列表（双 Claw 共用）。
|
||||
|
||||
返回 `_resolve_market_skills` 的结构：
|
||||
[{id, name, description, skill_content, category, func_name, parameters}]
|
||||
"""
|
||||
limit = limit if limit is not None else _env_int("CAP_SKILL_LIMIT", 12)
|
||||
if limit <= 0:
|
||||
return []
|
||||
|
||||
# 延迟导入：避免 app.core ←→ app.api 循环依赖
|
||||
from app.api.chat_skill import _resolve_market_skills
|
||||
from app.models import AppInstallation, MarketApp
|
||||
|
||||
pool: list = []
|
||||
source = "empty"
|
||||
|
||||
# 1) 用户显式安装
|
||||
if user_id:
|
||||
try:
|
||||
app_ids = [
|
||||
i.app_id
|
||||
for i in db.query(AppInstallation)
|
||||
.filter(AppInstallation.user_id == user_id)
|
||||
.all()
|
||||
]
|
||||
if app_ids:
|
||||
ids = [
|
||||
a.id
|
||||
for a in db.query(MarketApp)
|
||||
.filter(
|
||||
MarketApp.id.in_(app_ids),
|
||||
MarketApp.type == "skill",
|
||||
MarketApp.status == "published",
|
||||
)
|
||||
.all()
|
||||
]
|
||||
if ids:
|
||||
pool = _resolve_market_skills(db, ids, user_id)
|
||||
source = "installed"
|
||||
except Exception as e:
|
||||
logger.warning("[cap] 安装技能发现失败(降级继续): %s", e)
|
||||
|
||||
# 2) 降级：平台已发布全集
|
||||
if not pool and _env_flag("CAP_SKILL_FALLBACK_PUBLISHED", True):
|
||||
try:
|
||||
ids = [
|
||||
a.id
|
||||
for a in db.query(MarketApp)
|
||||
.filter(MarketApp.type == "skill", MarketApp.status == "published")
|
||||
.all()
|
||||
]
|
||||
if ids:
|
||||
pool = _resolve_market_skills(db, ids, user_id or "")
|
||||
source = "published"
|
||||
except Exception as e:
|
||||
logger.warning("[cap] published 技能发现失败: %s", e)
|
||||
|
||||
# 3) 预算裁剪
|
||||
picked = rank_by_query(pool, query, limit)
|
||||
logger.info(
|
||||
"[cap] skills claw=%s source=%s pool=%d picked=%d",
|
||||
claw,
|
||||
source,
|
||||
len(pool),
|
||||
len(picked),
|
||||
)
|
||||
return picked
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# MCP 发现
|
||||
# --------------------------------------------------------------------------
|
||||
def discover_mcp_servers(
|
||||
db,
|
||||
explicit_ids: Iterable[str] | None = None,
|
||||
limit: int | None = None,
|
||||
claw: str = "web",
|
||||
) -> list:
|
||||
"""发现当前请求可用的 MCP 服务连接（双 Claw 共用）。
|
||||
|
||||
Web Claw 传 explicit_ids（用户在 UI 勾选的）；
|
||||
IM Bot Claw 无勾选通道 → 传 None，自动挂载平台已启用的 MCP。
|
||||
返回 [{id, name, description, transport, endpoint, url, headers}]
|
||||
"""
|
||||
limit = limit if limit is not None else _env_int("CAP_MCP_LIMIT", 8)
|
||||
if limit <= 0:
|
||||
return []
|
||||
|
||||
from app.api.chat_mcp import _list_enabled_mcp_servers, _resolve_active_mcp_servers
|
||||
|
||||
ids = [i for i in (explicit_ids or []) if i]
|
||||
if ids:
|
||||
out = _resolve_active_mcp_servers(db, ids)[:limit]
|
||||
logger.info("[cap] mcp claw=%s source=explicit picked=%d", claw, len(out))
|
||||
return out
|
||||
|
||||
if not _env_flag("CAP_MCP_AUTO", True):
|
||||
return []
|
||||
|
||||
out = _list_enabled_mcp_servers(db)[:limit]
|
||||
logger.info("[cap] mcp claw=%s source=auto picked=%d", claw, len(out))
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# MCP → function definitions
|
||||
# --------------------------------------------------------------------------
|
||||
def _safe_name(raw: str, maxlen: int) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_-]", "_", (raw or "")[:maxlen])
|
||||
|
||||
|
||||
async def build_mcp_tool_defs(servers: list, idx_start: int = 0) -> tuple:
|
||||
"""把 MCP 服务列表展开为 (function_definitions, tool_map)。
|
||||
|
||||
命名规则与 Web Claw 现有实现严格一致：`mcp__{server}__{tool}__{idx}`，
|
||||
保证两端产生的工具名可互相对照、可被同一套审批/统计逻辑识别。
|
||||
|
||||
返回:
|
||||
function_definitions: [{name, description, parameters}]
|
||||
tool_map: {func_name: {server_id, name, server_name}}
|
||||
"""
|
||||
from app.mcp_client import list_mcp_tools
|
||||
|
||||
defs: list = []
|
||||
tool_map: dict = {}
|
||||
|
||||
for offset, srv in enumerate(servers or []):
|
||||
idx = idx_start + offset
|
||||
try:
|
||||
tools = await list_mcp_tools(srv)
|
||||
except Exception as e:
|
||||
logger.warning("[cap] MCP 拉取工具失败 [%s]: %s", srv.get("name"), e)
|
||||
continue
|
||||
for t in tools or []:
|
||||
sname = _safe_name(srv.get("name") or "unknown", 30)
|
||||
tname = _safe_name(t.get("name") or "tool", 50)
|
||||
fname = f"mcp__{sname}__{tname}__{idx}"
|
||||
tool_map[fname] = {
|
||||
"server_id": srv.get("id"),
|
||||
"name": t.get("name"),
|
||||
"server_name": srv.get("name"),
|
||||
}
|
||||
defs.append(
|
||||
{
|
||||
"name": fname,
|
||||
"description": (
|
||||
f"[MCP:{srv.get('name')}] {t.get('description') or t.get('name') or ''}"
|
||||
)[:900],
|
||||
"parameters": t.get("inputSchema")
|
||||
or {"type": "object", "properties": {}},
|
||||
}
|
||||
)
|
||||
return defs, tool_map
|
||||
@@ -83,6 +83,11 @@ def _create_kwargs(request_body: dict, stream: bool = False) -> dict:
|
||||
if tools:
|
||||
kw["tools"] = tools
|
||||
kw["tool_choice"] = request_body.get("tool_choice", "auto")
|
||||
# 2026-07-30: 将未在白名单的额外字段（如 enable_thinking）作为 extra_body 传递
|
||||
_whitelist = {"model", "messages", "stream", "temperature", "top_p", "max_tokens", "stop", "tools", "tool_choice"}
|
||||
_extras = {k: v for k, v in request_body.items() if k not in _whitelist}
|
||||
if _extras:
|
||||
kw["extra_body"] = _extras
|
||||
return kw
|
||||
|
||||
|
||||
@@ -223,29 +228,85 @@ def build_openai_client_sync(api_url, api_key, *, proxy=None, timeout=120, auth_
|
||||
return OpenAI(base_url=base, api_key=api_key or "EMPTY", timeout=timeout, http_client=http_client, default_headers=_hdrs if _hdrs else None)
|
||||
|
||||
|
||||
def chat_complete(api_url, api_key, request_body, *, proxy=None, timeout=120):
|
||||
"""同步版（与 achat_complete 行为/返回形状完全一致），供非 async 上下文使用。"""
|
||||
client = build_openai_client_sync(api_url, api_key, proxy=proxy, timeout=timeout, auth_header=auth_header)
|
||||
with client:
|
||||
resp = client.chat.completions.create(**_create_kwargs(request_body))
|
||||
msg = resp.choices[0].message
|
||||
return {
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": msg.content or "",
|
||||
"tool_calls": _normalize_tool_calls(getattr(msg, "tool_calls", None)),
|
||||
"reasoning_content": getattr(msg, "reasoning_content", None),
|
||||
}
|
||||
def chat_complete(api_url, api_key, request_body, *, proxy=None, timeout=120, auth_header=None):
|
||||
"""同步版（与 achat_complete 行为/返回形状完全一致），供非 async 上下文使用。
|
||||
|
||||
2026-07-30 重写：直接使用 requests 替代 OpenAI SDK/httpx。
|
||||
OpenAI SDK 的 httpx 连接池在慢长轮询（如 iLink QR 状态 12-19s）后会出现
|
||||
Connection error，已实测复现。requests 库没有这个问题。
|
||||
"""
|
||||
import requests
|
||||
import base64
|
||||
import random
|
||||
body = dict(request_body)
|
||||
body["stream"] = False
|
||||
# enable_thinking 等额外字段已在 request_body 中
|
||||
|
||||
# Headers: Bearer token + X-WECHAT-UIN (兼容 iLink)
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {api_key}" if api_key else "Bearer empty",
|
||||
}
|
||||
if auth_header and ":" in auth_header:
|
||||
_k, _v = auth_header.split(":", 1)
|
||||
headers[_k.strip()] = _v.strip()
|
||||
else:
|
||||
# X-WECHAT-UIN 防重放
|
||||
headers["X-WECHAT-UIN"] = base64.b64encode(str(random.randint(0, 2**32-1)).encode()).decode()
|
||||
|
||||
# 2026-07-31 修复根因: proxy 为 None 时传 proxies={} 而非 None。
|
||||
# requests 在 proxies=None 时会继续读取环境 HTTP_PROXY(trust_env 默认 True),
|
||||
# 而 uvicorn 进程(run.sh)带有 HTTP_PROXY=172.18.0.1:7890, 该代理无法解析
|
||||
# Docker 内部域名(如 qihu-aiflow-litellm), 导致所有内部模型调用静默失败、返回空。
|
||||
# 显式 proxies={} + trust_env=False 可完全绕过环境代理, 使 _proxy_for_url 的
|
||||
# 决策(返回 None=不代理)真正生效。
|
||||
_proxies = {"http": proxy, "https": proxy} if proxy else {}
|
||||
_session = requests.Session()
|
||||
_session.trust_env = False
|
||||
last_body = {}
|
||||
for attempt in range(2):
|
||||
try:
|
||||
resp = _session.post(
|
||||
api_url.rstrip("/"),
|
||||
headers=headers,
|
||||
json=body,
|
||||
proxies=_proxies,
|
||||
timeout=timeout,
|
||||
)
|
||||
try:
|
||||
last_body = resp.json() if resp.content else {}
|
||||
except Exception:
|
||||
last_body = {}
|
||||
if resp.ok:
|
||||
import logging as _l6
|
||||
_l6.warning(f'[chat_complete] body keys={list(last_body.keys()) if isinstance(last_body, dict) else type(last_body)}, content_len={len(resp.content)}, content_preview={str(resp.content)[:200]}')
|
||||
return _shape_response(last_body)
|
||||
except Exception:
|
||||
pass
|
||||
return {"choices": [{"message": {"content": "", "tool_calls": None, "reasoning_content": None}}]}
|
||||
|
||||
|
||||
def _shape_response(body: dict) -> dict:
|
||||
"""Normalize OpenAI-compatible response to standard shape."""
|
||||
choices = body.get("choices", [])
|
||||
import logging as _l7
|
||||
if choices:
|
||||
msg = choices[0].get("message", {})
|
||||
result = {
|
||||
"choices": [{
|
||||
"message": {
|
||||
"content": msg.get("content") or "",
|
||||
"tool_calls": _normalize_tool_calls(msg.get("tool_calls")),
|
||||
"reasoning_content": msg.get("reasoning_content") or "",
|
||||
}
|
||||
],
|
||||
"usage": (resp.usage.model_dump() if resp.usage else {}),
|
||||
}]
|
||||
}
|
||||
_l7.warning(f'[_shape] content_len={len(result["choices"][0]["message"]["content"])}, keys={list(result["choices"][0]["message"].keys())}')
|
||||
return result
|
||||
_l7.warning(f'[_shape] NO choices in body, keys={list(body.keys())[:5]}')
|
||||
return body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 嵌入（Embeddings）：OpenAI 兼容 /v1/embeddings，收敛散落的裸 httpx 嵌入调用
|
||||
# ---------------------------------------------------------------------------
|
||||
def _embed_kwargs(input, model):
|
||||
return {"model": model, "input": input if isinstance(input, list) else [input]}
|
||||
|
||||
|
||||
Reference in new issue
Block a user