feat(observability): Phase2.3 媒体/LLM 可观测面板(后端归因聚合+前端页) 并修复 token_usage_logs trace_id 写入(chat_service/im_llm) [v2026.07.19.5]
CI / gate (push) Has been cancelled

This commit is contained in:
QihuAI Design committed 2026-07-19 10:18:14 +08:00
1 parent 491f31cead
commit 2969544d54
9 files changed
+3357 -3079

No files matched your search

@@ -0,0 +1,103 @@
"""调用可观测面板后端 API(媒体/LLM 调用归因)。
路由前缀: /app/api/admin/observability
数据来源: token_usage_logs(Phase 2.2 已加 trace_id 列,可经 X-Trace-Id 关联请求链路)。
"""
from datetime import datetime, timedelta
from fastapi import APIRouter, Query, Depends
from ...database import get_db
from ..auth import get_current_user
from sqlalchemy import text
from sqlalchemy.orm import Session
router = APIRouter()
@router.get("/summary")
async def observability_summary(
days: int = Query(30, description="统计天数"),
db: Session = Depends(get_db),
user=Depends(get_current_user),
):
"""媒体/LLM 调用归因:总览 + 按模型 + 按用户 + 每日趋势 + 最近调用(trace_id 下钻)。"""
since = datetime.now() - timedelta(days=days)
totals = db.execute(text("""
SELECT COUNT(*),
COALESCE(SUM(total_tokens), 0),
COALESCE(SUM(cost), 0),
COUNT(DISTINCT user_id)
FROM token_usage_logs WHERE created_at >= :since
"""), {"since": since}).fetchone()
by_model = db.execute(text("""
SELECT model_name, COUNT(*), COALESCE(SUM(total_tokens), 0), COALESCE(SUM(cost), 0)
FROM token_usage_logs WHERE created_at >= :since
GROUP BY model_name ORDER BY COALESCE(SUM(total_tokens), 0) DESC
"""), {"since": since}).fetchall()
by_user = db.execute(text("""
SELECT t.user_id, u.username, COUNT(*), COALESCE(SUM(t.total_tokens), 0), COALESCE(SUM(t.cost), 0)
FROM token_usage_logs t LEFT JOIN users u ON u.id = t.user_id
WHERE t.created_at >= :since
GROUP BY t.user_id, u.username ORDER BY COALESCE(SUM(t.total_tokens), 0) DESC LIMIT 20
"""), {"since": since}).fetchall()
daily = db.execute(text("""
SELECT DATE(created_at) AS d, COALESCE(SUM(total_tokens), 0), COALESCE(SUM(cost), 0), COUNT(*)
FROM token_usage_logs WHERE created_at >= :since
GROUP BY DATE(created_at) ORDER BY d
"""), {"since": since}).fetchall()
recent = db.execute(text("""
SELECT trace_id, model_name, total_tokens, cost, created_at
FROM token_usage_logs WHERE created_at >= :since
ORDER BY created_at DESC LIMIT 50
"""), {"since": since}).fetchall()
return {
"totals": {
"calls": int(totals[0] or 0),
"tokens": int(totals[1] or 0),
"cost": float(totals[2] or 0),
"users": int(totals[3] or 0),
},
"by_model": [
{
"model_name": m[0] or "unknown",
"calls": int(m[1] or 0),
"tokens": int(m[2] or 0),
"cost": float(m[3] or 0),
}
for m in by_model
],
"by_user": [
{
"user_id": u[0],
"username": u[1] or u[0],
"calls": int(u[2] or 0),
"tokens": int(u[3] or 0),
"cost": float(u[4] or 0),
}
for u in by_user
],
"daily": [
{
"date": str(d[0]),
"tokens": int(d[1] or 0),
"cost": float(d[2] or 0),
"calls": int(d[3] or 0),
}
for d in daily
],
"recent": [
{
"trace_id": r[0],
"model_name": r[1] or "unknown",
"tokens": int(r[2] or 0),
"cost": float(r[3] or 0),
"created_at": r[4].strftime("%Y-%m-%d %H:%M:%S") if r[4] else None,
}
for r in recent
],
}
+3 -1
View File
@@ -16,7 +16,7 @@ import httpx
import os
import logging
from ..database import get_db, SessionLocal
from app.core.observability import span
from app.core.observability import span, get_trace_id
from ..models import (
@@ -1743,6 +1743,8 @@ async def _send_message_core(
conversation_id=conv_id,
trace_id=get_trace_id(),
+234 -232
View File
@@ -1,232 +1,234 @@
"""
IM Bot 真实 LLM 调用。
原实现位于 channels.py(通用频道管理模块),该模块已移除。
为保持 IM Bot(微信机器人等)正常生成回复,将 _call_real_llm 迁移至此独立模块。
行为(对齐 AI 聊天):
- Issue 7:图像/视频/语音意图优先走平台媒体模型(与 AI 聊天同一套 Auto 路由),
不再对"制作图片"这类请求只回文本。
- Issue 8:默认 Auto 模型;每轮先自动发现本账号已安装且平台已部署的技能,
多个适用时由模型自行判断选 1 个最优,通过 function calling 执行。
- 2026-07-15:修复 trust_env=False + proxy 处理,避免容器坏代理导致调用失败;
媒体生成前先经 _optimize_media_prompt 精炼提示词,提升 AGNES 生成质量。
"""
import asyncio
import json
import logging
import re as _re
from datetime import datetime, timezone, timedelta
from typing import Optional
from urllib.parse import urlparse as _urlparse
import httpx
from sqlalchemy.orm import Session
from ..capability_registry import resolve_auto_model
from ..models import AIModel, ModelProvider, TokenUsageLog, gen_uuid
logger = logging.getLogger(__name__)
# 香港出网代理(实测对 AGNES 延迟更低,2026-07-15 用户实测),与 chat.py 同值。
_HK_PROXY = "http://qyyfw:XHR521yhf@103.242.14.208:11495"
def _proxy_for_url(url: str) -> str | None:
"""按目标域名选择是否走代理(与 chat.py 同逻辑)。
2026-07-15 调整:AGNES 改走香港代理(实测延迟更低),不再直连。
国内域名(尊重 NO_PROXY)直连,更稳定。
"""
import os
host = (_urlparse(url).hostname or "").lower()
if "agnes-ai.com" in host or "agnes-ai.space" in host:
return _HK_PROXY
if "qihuyun.com" in host:
return None
_no_proxy = [h.strip().lower() for h in os.environ.get("NO_PROXY", "").split(",") if h.strip()]
for _np in _no_proxy:
if host == _np or host.endswith("." + _np) or _np in host:
return None
return os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or "http://172.18.0.1:7890"
def _run_async(coro, timeout=None):
"""在同步上下文中运行协程,兼容「无事件循环」与「已有事件循环」两种场景。
IM 个人微信在独立线程中调用(无事件循环)→ asyncio.run;
IM 多平台 Webhook 可能在 FastAPI 协程内调用(已有事件循环)→ 丢到线程池执行。
timeout: 可选,秒数;超时抛出 asyncio.TimeoutError(由调用方捕获)。
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if timeout is not None:
async def _timed():
return await asyncio.wait_for(coro, timeout=timeout)
target = _timed()
else:
target = coro
if loop is not None and loop.is_running():
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(1) as ex:
return ex.submit(lambda: asyncio.run(target)).result(timeout=(timeout + 10) if timeout else None)
return asyncio.run(target)
def _log_media_usage(db: Session, user_id: str, ai_model, media_type: str, prompt: str):
"""IM Bot 媒体生成记账:把图片/视频消耗写入 TokenUsageLog,供驾驶舱统计。
此前 IM Bot 路径完全不写 TokenUsageLog,导致微信生成的图片/视频在驾驶舱里“查无此项”。
媒体 API 不返回 token 用量,这里用与 chat.py _generate_media_task 一致的估算。
"""
if not user_id or not ai_model:
return
try:
base = 1200 if media_type == "video" else 400
est = base + (len(prompt or "") // 4)
db.add(TokenUsageLog(
id=gen_uuid(),
user_id=user_id,
model_id=ai_model.id,
model_name=ai_model.name,
prompt_tokens=est,
completion_tokens=0,
total_tokens=est,
created_at=datetime.now(timezone(timedelta(hours=8))).replace(tzinfo=None),
))
db.commit()
except Exception as e:
try:
db.rollback()
except Exception:
pass
logger.warning(f"[im_llm] 媒体 Token 记账失败(忽略): {e}")
def _call_real_llm(db: Session, user=None, prompt: str = "", platform: str = "im", user_id: str = None) -> "str | dict":
"""调用七虎AI 生成回复,供 IM Bot 复用。支持媒体意图路由与已安装技能自动发现。"""
try:
from app.api.chat import (
_generate_media,
detect_media_intent,
_discover_installed_skills,
_build_system_prompt,
_execute_skill_tool,
)
# ===== 1) 媒体意图路由(Issue 7)=====
intent = detect_media_intent(prompt)
if intent:
ai_model = resolve_auto_model(intent, db)
if ai_model:
try:
result = _run_async(_generate_media(db, ai_model, prompt, user_id), timeout=300)
except Exception:
# 任何未预期异常(如 AGNES 连接重置)统一兜底为友好提示,
# 绝不直接把原始异常(如 "Server disconnected")甩给用户
result = "⚠️ 媒体生成暂时失败(连接或服务波动),请稍后重试。"
if result:
# 媒体生成成功:返回结构化对象,交由 im_bot_user 在微信内联发送图片/视频
s = result.strip()
if s.startswith("<img") or s.startswith("<video"):
m = _re.search(r'src="([^"]+)"', s)
url = m.group(1) if m else ""
media_type = "image" if s.startswith("<img") else "video"
if not url:
return "(生成完成,但未能提取媒体地址)"
# IM Bot 路径此前完全不记账 → 补写媒体消耗,驾驶舱才看得到图片/视频数据
_log_media_usage(db, user_id, ai_model, media_type, prompt)
return {"__media__": True, "media_type": media_type, "url": url}
# 媒体生成失败:_generate_media 已返回友好中文(如限频/队列繁忙),
# 直接透传,绝不把原始提示词/标签/JSON 透传给用户
if s.startswith("⚠️") or s.startswith("(媒体"):
return s
return "(媒体生成未能返回有效内容,请重试)"
# ===== 2) 文本回复 + 技能自动发现(Issue 8)=====
ai_model = resolve_auto_model("text", db)
if not ai_model:
return "(未配置默认文本模型,无法生成回复)"
provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first()
api_key = ai_model.api_key or (provider.api_key if provider else None)
api_url = ai_model.endpoint
if not api_url and provider and provider.api_base_url:
api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions"
if not (api_key and api_url):
return "(模型未配置 API Key 或地址,无法生成回复)"
# 自动发现本账号已安装且平台已部署的技能
skills = []
if user_id:
try:
skills = _discover_installed_skills(db, user_id)[:15]
except Exception:
skills = []
system_prompt, function_definitions = _build_system_prompt(
skills=skills,
agents=[],
kb_context="",
user_content=prompt,
mcp=[],
auto_discovered=bool(skills),
)
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt},
]
max_rounds = 3
ai_content = ""
for _ in range(max_rounds):
request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048}
if function_definitions and skills:
request_body["tools"] = [
{"type": "function", "function": fd} for fd in function_definitions
]
request_body["tool_choice"] = "auto"
with httpx.Client(
timeout=ai_model.timeout or 120,
proxy=_proxy_for_url(api_url),
trust_env=False,
) as client:
resp = client.post(
api_url,
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
json=request_body,
)
if resp.status_code != 200:
return f"(AI 模型调用失败 HTTP {resp.status_code})"
data = resp.json()
message = data.get("choices", [{}])[0].get("message", {})
tool_calls = message.get("tool_calls", [])
if tool_calls and skills:
messages.append({
"role": "assistant",
"content": message.get("content") or "",
"tool_calls": tool_calls,
})
for tc in tool_calls:
func_name = tc.get("function", {}).get("name", "")
try:
func_args = json.loads(tc.get("function", {}).get("arguments", "{}"))
except Exception:
func_args = {"input": tc.get("function", {}).get("arguments", "")}
tool_result = _execute_skill_tool(db, func_name, func_args, skills, prompt)
messages.append({
"role": "tool",
"tool_call_id": tc.get("id", "call_1"),
"content": tool_result,
})
continue
ai_content = message.get("content", "")
break
return ai_content or "(AI 回复为空)"
except Exception as e:
return f"(AI 回复生成失败:{e})"
"""
IM Bot 真实 LLM 调用。
原实现位于 channels.py(通用频道管理模块),该模块已移除。
为保持 IM Bot(微信机器人等)正常生成回复,将 _call_real_llm 迁移至此独立模块。
行为(对齐 AI 聊天):
- Issue 7:图像/视频/语音意图优先走平台媒体模型(与 AI 聊天同一套 Auto 路由),
不再对"制作图片"这类请求只回文本。
- Issue 8:默认 Auto 模型;每轮先自动发现本账号已安装且平台已部署的技能,
多个适用时由模型自行判断选 1 个最优,通过 function calling 执行。
- 2026-07-15:修复 trust_env=False + proxy 处理,避免容器坏代理导致调用失败;
媒体生成前先经 _optimize_media_prompt 精炼提示词,提升 AGNES 生成质量。
"""
import asyncio
import json
import logging
import re as _re
from datetime import datetime, timezone, timedelta
from typing import Optional
from urllib.parse import urlparse as _urlparse
import httpx
from sqlalchemy.orm import Session
from ..capability_registry import resolve_auto_model
from ..models import AIModel, ModelProvider, TokenUsageLog, gen_uuid
from app.core.observability import get_trace_id
logger = logging.getLogger(__name__)
# 香港出网代理(实测对 AGNES 延迟更低,2026-07-15 用户实测),与 chat.py 同值。
_HK_PROXY = "http://qyyfw:XHR521yhf@103.242.14.208:11495"
def _proxy_for_url(url: str) -> str | None:
"""按目标域名选择是否走代理(与 chat.py 同逻辑)。
2026-07-15 调整:AGNES 改走香港代理(实测延迟更低),不再直连。
国内域名(尊重 NO_PROXY)直连,更稳定。
"""
import os
host = (_urlparse(url).hostname or "").lower()
if "agnes-ai.com" in host or "agnes-ai.space" in host:
return _HK_PROXY
if "qihuyun.com" in host:
return None
_no_proxy = [h.strip().lower() for h in os.environ.get("NO_PROXY", "").split(",") if h.strip()]
for _np in _no_proxy:
if host == _np or host.endswith("." + _np) or _np in host:
return None
return os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or "http://172.18.0.1:7890"
def _run_async(coro, timeout=None):
"""在同步上下文中运行协程,兼容「无事件循环」与「已有事件循环」两种场景。
IM 个人微信在独立线程中调用(无事件循环)→ asyncio.run;
IM 多平台 Webhook 可能在 FastAPI 协程内调用(已有事件循环)→ 丢到线程池执行。
timeout: 可选,秒数;超时抛出 asyncio.TimeoutError(由调用方捕获)。
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if timeout is not None:
async def _timed():
return await asyncio.wait_for(coro, timeout=timeout)
target = _timed()
else:
target = coro
if loop is not None and loop.is_running():
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(1) as ex:
return ex.submit(lambda: asyncio.run(target)).result(timeout=(timeout + 10) if timeout else None)
return asyncio.run(target)
def _log_media_usage(db: Session, user_id: str, ai_model, media_type: str, prompt: str):
"""IM Bot 媒体生成记账:把图片/视频消耗写入 TokenUsageLog,供驾驶舱统计。
此前 IM Bot 路径完全不写 TokenUsageLog,导致微信生成的图片/视频在驾驶舱里“查无此项”。
媒体 API 不返回 token 用量,这里用与 chat.py _generate_media_task 一致的估算。
"""
if not user_id or not ai_model:
return
try:
base = 1200 if media_type == "video" else 400
est = base + (len(prompt or "") // 4)
db.add(TokenUsageLog(
id=gen_uuid(),
user_id=user_id,
model_id=ai_model.id,
model_name=ai_model.name,
trace_id=get_trace_id(),
prompt_tokens=est,
completion_tokens=0,
total_tokens=est,
created_at=datetime.now(timezone(timedelta(hours=8))).replace(tzinfo=None),
))
db.commit()
except Exception as e:
try:
db.rollback()
except Exception:
pass
logger.warning(f"[im_llm] 媒体 Token 记账失败(忽略): {e}")
def _call_real_llm(db: Session, user=None, prompt: str = "", platform: str = "im", user_id: str = None) -> "str | dict":
"""调用七虎AI 生成回复,供 IM Bot 复用。支持媒体意图路由与已安装技能自动发现。"""
try:
from app.api.chat import (
_generate_media,
detect_media_intent,
_discover_installed_skills,
_build_system_prompt,
_execute_skill_tool,
)
# ===== 1) 媒体意图路由(Issue 7)=====
intent = detect_media_intent(prompt)
if intent:
ai_model = resolve_auto_model(intent, db)
if ai_model:
try:
result = _run_async(_generate_media(db, ai_model, prompt, user_id), timeout=300)
except Exception:
# 任何未预期异常(如 AGNES 连接重置)统一兜底为友好提示,
# 绝不直接把原始异常(如 "Server disconnected")甩给用户
result = "⚠️ 媒体生成暂时失败(连接或服务波动),请稍后重试。"
if result:
# 媒体生成成功:返回结构化对象,交由 im_bot_user 在微信内联发送图片/视频
s = result.strip()
if s.startswith("<img") or s.startswith("<video"):
m = _re.search(r'src="([^"]+)"', s)
url = m.group(1) if m else ""
media_type = "image" if s.startswith("<img") else "video"
if not url:
return "(生成完成,但未能提取媒体地址)"
# IM Bot 路径此前完全不记账 → 补写媒体消耗,驾驶舱才看得到图片/视频数据
_log_media_usage(db, user_id, ai_model, media_type, prompt)
return {"__media__": True, "media_type": media_type, "url": url}
# 媒体生成失败:_generate_media 已返回友好中文(如限频/队列繁忙),
# 直接透传,绝不把原始提示词/标签/JSON 透传给用户
if s.startswith("⚠️") or s.startswith("(媒体"):
return s
return "(媒体生成未能返回有效内容,请重试)"
# ===== 2) 文本回复 + 技能自动发现(Issue 8)=====
ai_model = resolve_auto_model("text", db)
if not ai_model:
return "(未配置默认文本模型,无法生成回复)"
provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first()
api_key = ai_model.api_key or (provider.api_key if provider else None)
api_url = ai_model.endpoint
if not api_url and provider and provider.api_base_url:
api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions"
if not (api_key and api_url):
return "(模型未配置 API Key 或地址,无法生成回复)"
# 自动发现本账号已安装且平台已部署的技能
skills = []
if user_id:
try:
skills = _discover_installed_skills(db, user_id)[:15]
except Exception:
skills = []
system_prompt, function_definitions = _build_system_prompt(
skills=skills,
agents=[],
kb_context="",
user_content=prompt,
mcp=[],
auto_discovered=bool(skills),
)
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt},
]
max_rounds = 3
ai_content = ""
for _ in range(max_rounds):
request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048}
if function_definitions and skills:
request_body["tools"] = [
{"type": "function", "function": fd} for fd in function_definitions
]
request_body["tool_choice"] = "auto"
with httpx.Client(
timeout=ai_model.timeout or 120,
proxy=_proxy_for_url(api_url),
trust_env=False,
) as client:
resp = client.post(
api_url,
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
json=request_body,
)
if resp.status_code != 200:
return f"(AI 模型调用失败 HTTP {resp.status_code})"
data = resp.json()
message = data.get("choices", [{}])[0].get("message", {})
tool_calls = message.get("tool_calls", [])
if tool_calls and skills:
messages.append({
"role": "assistant",
"content": message.get("content") or "",
"tool_calls": tool_calls,
})
for tc in tool_calls:
func_name = tc.get("function", {}).get("name", "")
try:
func_args = json.loads(tc.get("function", {}).get("arguments", "{}"))
except Exception:
func_args = {"input": tc.get("function", {}).get("arguments", "")}
tool_result = _execute_skill_tool(db, func_name, func_args, skills, prompt)
messages.append({
"role": "tool",
"tool_call_id": tc.get("id", "call_1"),
"content": tool_result,
})
continue
ai_content = message.get("content", "")
break
return ai_content or "(AI 回复为空)"
except Exception as e:
return f"(AI 回复生成失败:{e})"
+250 -249
View File
@@ -1,249 +1,250 @@
"""
Qihu-AIFlow FastAPI 主应用
完整后端API,监听18080端口,所有接口以 /app 为前缀
"""
import sys
import os
import time
import uuid
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from starlette.middleware.base import BaseHTTPMiddleware
from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from app.api import auth, chat, knowledge, market, token_usage, payment, profile, notifications, assets, evolution
from app.casdoor_auth import router as casdoor_router
from app.api.admin import dashboard, users, models, settings, payment_admin, services, membership_admin, ads, market_admin, logs, skills_admin, agents_admin, knowledge_admin, workflow_admin, storage_admin, automation_admin, mcp_admin, developer_admin, apps_admin, distro_admin, code_exec_admin, privacy_admin, im_bot_admin, promotion_admin, achievement_admin, tenant_admin, billing_admin, analytics_admin
from app.api.im_bot_user import router as im_bot_user_router
from app.api.im_bot_platform import router as im_bot_platform_router
from app.api.portal import user_portal
from app.api.admin.developer_tools_admin import router as developer_tools_router
from app.api.admin.mcp_market_admin import router as mcp_market_router
from app.api.admin import asset_admin, achievement_admin
from app.config import setup_file_logging, SLOW_REQUEST_THRESHOLD
from app.core.observability import set_trace_id
from app.database import SessionLocal
from app.models import OperationLog as OperationLogModel, TokenUsageLog as TokenUsageLogModel
# 导入平台适配器触发注册(AstrBot借鉴优化方案 P1)
import app.platform_adapters # noqa: F401 - 触发所有平台适配器装饰器注册
# 初始化文件日志(按天轮转)
access_logger = setup_file_logging()
logger = logging.getLogger("aiflow")
class APITrackingMiddleware(BaseHTTPMiddleware):
"""API调用统计中间件 - 记录请求耗时,标记慢请求"""
async def dispatch(self, request: Request, call_next):
request_id = str(uuid.uuid4())
set_trace_id(request_id)
start_time = time.time()
path = request.url.path
method = request.method
# 跳过健康检查和静态资源
skip_paths = ["/app/api/health", "/favicon.ico"]
should_skip = any(path.startswith(p) for p in skip_paths)
try:
response = await call_next(request)
elapsed = time.time() - start_time
status_code = response.status_code
response.headers["X-Trace-Id"] = request_id
if not should_skip:
access_logger.info(
f"[{request_id}] {method} {path} -> {status_code} ({elapsed:.3f}s)"
)
# 异步记录API调用统计
try:
db = SessionLocal()
try:
_record_api_call(db, method, path, status_code, elapsed)
finally:
db.close()
except Exception as e:
logger.warning(f"API统计记录失败: {e}")
return response
except Exception as e:
elapsed = time.time() - start_time
if not should_skip:
access_logger.info(
f"[{request_id}] {method} {path} -> ERROR ({elapsed:.3f}s): {str(e)}"
)
raise
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
await super().__call__(scope, receive, send)
def _record_api_call(db, method: str, path: str, status_code: int, elapsed: float):
"""记录API调用到数据库统计表"""
# 简化路径以减少基数(去掉动态ID)
parts = path.split("/")
simplified_parts = []
for part in parts:
# 将UUID样式的路径段替换为{id}
if len(part) == 36 and part.count("-") == 4:
simplified_parts.append("{id}")
# 将纯数字路径段替换为{id}
elif part.isdigit():
simplified_parts.append("{id}")
else:
simplified_parts.append(part)
simplified_path = "/".join(simplified_parts)
# 统计记录通过数据库实现,简化版记录到日志
is_error = status_code >= 400
is_slow = elapsed >= SLOW_REQUEST_THRESHOLD
if is_slow:
logger.warning(
f"慢请求: {method} {simplified_path} -> {status_code} ({elapsed:.3f}s, 阈值: {SLOW_REQUEST_THRESHOLD}s)"
)
@asynccontextmanager
async def lifespan(app: FastAPI):
logger.info("Qihu-AIFlow v2.0.0 启动中...")
# 启动后台 Evolution Pipeline(知识库图谱已存在的用户配置自动恢复运行)
try:
from app.services.evolution_service import EvolutionWatcherManager
from app.models import UserEvolutionConfig
db = SessionLocal()
try:
configs = db.query(UserEvolutionConfig).filter(
UserEvolutionConfig.watcher_status == "running"
).all()
if configs:
watcher = EvolutionWatcherManager()
for cfg in configs:
try:
watcher.start_user_evolution(cfg.user_id, lambda: SessionLocal())
logger.info(f"[Evolution] 恢复用户 {cfg.user_id} 的进化Pipeline")
except Exception as e:
logger.warning(f"[Evolution] 恢复用户 {cfg.user_id} 失败: {e}")
logger.info(f"[Evolution] 已恢复 {len(configs)} 个用户的进化Pipeline")
else:
logger.info("[Evolution] 无待恢复的进化配置")
finally:
db.close()
except Exception as e:
logger.warning(f"[Evolution] Pipeline自动启动异常(不影响正常服务): {e}")
# 启动Redis事件总线
try:
from app.event_bus import get_event_bus
event_bus = get_event_bus()
event_bus.start(num_consumers=2)
logger.info("[EventBus] Redis事件总线已启动")
except Exception as e:
logger.warning(f"[EventBus] 启动失败(不影响正常服务): {e}")
logger.info("Qihu-AIFlow v2.0.0 启动完成")
yield
# 停止Redis事件总线
try:
from app.event_bus import get_event_bus
get_event_bus().stop()
logger.info("[EventBus] Redis事件总线已停止")
except Exception:
pass
# 优雅关闭:停止所有 Evolution Pipeline
try:
from app.services.evolution_service import EvolutionWatcherManager
EvolutionWatcherManager().shutdown_all()
logger.info("[Evolution] 所有Pipeline已停止")
except Exception:
pass
logger.info("Qihu-AIFlow 正在关闭...")
app = FastAPI(
title="Qihu-AIFlow - 智能自动化PaaS平台",
version="1.0.0",
lifespan=lifespan,
)
# API调用追踪中间件
app.add_middleware(APITrackingMiddleware)
# CORS配置
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# 前台 API
app.include_router(auth.router, prefix="/app/api/auth", tags=["认证"])
app.include_router(chat.router, prefix="/app/api/chat", tags=["AI聊天"])
app.include_router(knowledge.router, prefix="/app/api/knowledge", tags=["知识库"])
app.include_router(market.router, prefix="/app/api/market", tags=["应用市场"])
app.include_router(token_usage.router, prefix="/app/api/token", tags=["Token驾驶舱"])
app.include_router(payment.router, prefix="/app/api/payment", tags=["付费中心"])
app.include_router(profile.router, prefix="/app/api/profile", tags=["个人中心"])
app.include_router(notifications.router, prefix="/app/api/notifications", tags=["消息通知"])
app.include_router(assets.router, prefix="/app/api/assets", tags=["资产管理"])
app.include_router(evolution.router, prefix="/app/api/evolution", tags=["AI自进化"])
# 后台 API
app.include_router(dashboard.router, prefix="/app/api/admin/dashboard", tags=["后台-仪表盘"])
app.include_router(users.router, prefix="/app/api/admin/users", tags=["后台-用户管理"])
app.include_router(models.router, prefix="/app/api/admin/models", tags=["后台-模型管理"])
app.include_router(settings.router, prefix="/app/api/admin/settings", tags=["后台-网站设置"])
app.include_router(payment_admin.router, prefix="/app/api/admin/payment", tags=["后台-支付配置"])
app.include_router(services.router, prefix="/app/api/admin/services", tags=["后台-增值服务"])
app.include_router(membership_admin.router, prefix="/app/api/admin/membership", tags=["后台-会员管理"])
app.include_router(ads.router, prefix="/app/api/admin/ads", tags=["后台-广告管理"])
app.include_router(market_admin.router, prefix="/app/api/admin/market", tags=["后台-应用审核"])
app.include_router(logs.router, prefix="/app/api/admin/logs", tags=["后台-日志风控"])
app.include_router(skills_admin.router, prefix="/app/api/admin/skills", tags=["后台-技能管理"])
app.include_router(agents_admin.router, prefix="/app/api/admin/agents", tags=["后台-Agent管理"])
app.include_router(knowledge_admin.router, prefix="/app/api/admin/knowledge", tags=["后台-知识库管理"])
app.include_router(workflow_admin.router, prefix="/app/api/admin/workflows", tags=["后台-工作流编排"])
app.include_router(asset_admin.router, prefix="/app/api/admin/assets", tags=["后台-资产管理"])
app.include_router(storage_admin.router, prefix="/app/api/admin/storage", tags=["后台-存储配置"])
app.include_router(automation_admin.router, prefix="/app/api/admin/automation", tags=["后台-自动化任务"])
app.include_router(mcp_admin.router, prefix="/app/api/admin/mcp", tags=["后台-MCP服务"])
app.include_router(developer_admin.router, prefix="/app/api/admin/developer", tags=["后台-开发者API"])
app.include_router(developer_tools_router, prefix="/app/api/admin/developer-tools", tags=["dev-tools"])
app.include_router(apps_admin.router, prefix="/app/api/admin/apps", tags=["admin-ai-apps"])
app.include_router(mcp_market_router, prefix="/app/api/admin/mcp-market", tags=["mcp-market"])
app.include_router(code_exec_admin.router, prefix="/app/api/admin/code-exec", tags=["code-exec"])
app.include_router(distro_admin.router, prefix="/app/api/admin/distributions", tags=["distributions"])
app.include_router(privacy_admin.router, prefix="/app/api/admin/privacy", tags=["privacy"])
app.include_router(im_bot_admin.router, prefix="/app/api/admin/im-bot", tags=["im-bot"])
app.include_router(im_bot_user_router, prefix="/app/api/im/wechat", tags=["im-wechat-user"])
app.include_router(im_bot_platform_router, prefix="/app/api/im/platform", tags=["im-platform-user"])
app.include_router(promotion_admin.router, prefix="/app/api/admin/promotion", tags=["promotion"])
app.include_router(tenant_admin.router, prefix="/app/api/admin/tenant", tags=["tenant"])
app.include_router(billing_admin.router, prefix="/app/api/admin/billing", tags=["billing"])
app.include_router(analytics_admin.router, prefix="/app/api/admin/analytics", tags=["analytics"])
app.include_router(user_portal.router, prefix="/app/api/portal", tags=["portal"])
app.include_router(achievement_admin.router, prefix="/app/api/admin/achievement", tags=["achievement"])
app.include_router(casdoor_router, prefix="/app/api/auth", tags=["Casdoor SSO"])
@app.get("/app/api/health")
async def health_check():
return {"status": "ok", "service": "Qihu-AIFlow"}
"""
Qihu-AIFlow FastAPI 主应用
完整后端API,监听18080端口,所有接口以 /app 为前缀
"""
import sys
import os
import time
import uuid
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from starlette.middleware.base import BaseHTTPMiddleware
from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from app.api import auth, chat, knowledge, market, token_usage, payment, profile, notifications, assets, evolution
from app.casdoor_auth import router as casdoor_router
from app.api.admin import dashboard, users, models, settings, payment_admin, services, membership_admin, ads, market_admin, logs, skills_admin, agents_admin, knowledge_admin, workflow_admin, storage_admin, automation_admin, mcp_admin, developer_admin, apps_admin, distro_admin, code_exec_admin, privacy_admin, im_bot_admin, promotion_admin, achievement_admin, tenant_admin, billing_admin, analytics_admin, observability_admin
from app.api.im_bot_user import router as im_bot_user_router
from app.api.im_bot_platform import router as im_bot_platform_router
from app.api.portal import user_portal
from app.api.admin.developer_tools_admin import router as developer_tools_router
from app.api.admin.mcp_market_admin import router as mcp_market_router
from app.api.admin import asset_admin, achievement_admin
from app.config import setup_file_logging, SLOW_REQUEST_THRESHOLD
from app.core.observability import set_trace_id
from app.database import SessionLocal
from app.models import OperationLog as OperationLogModel, TokenUsageLog as TokenUsageLogModel
# 导入平台适配器触发注册(AstrBot借鉴优化方案 P1)
import app.platform_adapters # noqa: F401 - 触发所有平台适配器装饰器注册
# 初始化文件日志(按天轮转)
access_logger = setup_file_logging()
logger = logging.getLogger("aiflow")
class APITrackingMiddleware(BaseHTTPMiddleware):
"""API调用统计中间件 - 记录请求耗时,标记慢请求"""
async def dispatch(self, request: Request, call_next):
request_id = str(uuid.uuid4())
set_trace_id(request_id)
start_time = time.time()
path = request.url.path
method = request.method
# 跳过健康检查和静态资源
skip_paths = ["/app/api/health", "/favicon.ico"]
should_skip = any(path.startswith(p) for p in skip_paths)
try:
response = await call_next(request)
elapsed = time.time() - start_time
status_code = response.status_code
response.headers["X-Trace-Id"] = request_id
if not should_skip:
access_logger.info(
f"[{request_id}] {method} {path} -> {status_code} ({elapsed:.3f}s)"
)
# 异步记录API调用统计
try:
db = SessionLocal()
try:
_record_api_call(db, method, path, status_code, elapsed)
finally:
db.close()
except Exception as e:
logger.warning(f"API统计记录失败: {e}")
return response
except Exception as e:
elapsed = time.time() - start_time
if not should_skip:
access_logger.info(
f"[{request_id}] {method} {path} -> ERROR ({elapsed:.3f}s): {str(e)}"
)
raise
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
await super().__call__(scope, receive, send)
def _record_api_call(db, method: str, path: str, status_code: int, elapsed: float):
"""记录API调用到数据库统计表"""
# 简化路径以减少基数(去掉动态ID)
parts = path.split("/")
simplified_parts = []
for part in parts:
# 将UUID样式的路径段替换为{id}
if len(part) == 36 and part.count("-") == 4:
simplified_parts.append("{id}")
# 将纯数字路径段替换为{id}
elif part.isdigit():
simplified_parts.append("{id}")
else:
simplified_parts.append(part)
simplified_path = "/".join(simplified_parts)
# 统计记录通过数据库实现,简化版记录到日志
is_error = status_code >= 400
is_slow = elapsed >= SLOW_REQUEST_THRESHOLD
if is_slow:
logger.warning(
f"慢请求: {method} {simplified_path} -> {status_code} ({elapsed:.3f}s, 阈值: {SLOW_REQUEST_THRESHOLD}s)"
)
@asynccontextmanager
async def lifespan(app: FastAPI):
logger.info("Qihu-AIFlow v2.0.0 启动中...")
# 启动后台 Evolution Pipeline(知识库图谱已存在的用户配置自动恢复运行)
try:
from app.services.evolution_service import EvolutionWatcherManager
from app.models import UserEvolutionConfig
db = SessionLocal()
try:
configs = db.query(UserEvolutionConfig).filter(
UserEvolutionConfig.watcher_status == "running"
).all()
if configs:
watcher = EvolutionWatcherManager()
for cfg in configs:
try:
watcher.start_user_evolution(cfg.user_id, lambda: SessionLocal())
logger.info(f"[Evolution] 恢复用户 {cfg.user_id} 的进化Pipeline")
except Exception as e:
logger.warning(f"[Evolution] 恢复用户 {cfg.user_id} 失败: {e}")
logger.info(f"[Evolution] 已恢复 {len(configs)} 个用户的进化Pipeline")
else:
logger.info("[Evolution] 无待恢复的进化配置")
finally:
db.close()
except Exception as e:
logger.warning(f"[Evolution] Pipeline自动启动异常(不影响正常服务): {e}")
# 启动Redis事件总线
try:
from app.event_bus import get_event_bus
event_bus = get_event_bus()
event_bus.start(num_consumers=2)
logger.info("[EventBus] Redis事件总线已启动")
except Exception as e:
logger.warning(f"[EventBus] 启动失败(不影响正常服务): {e}")
logger.info("Qihu-AIFlow v2.0.0 启动完成")
yield
# 停止Redis事件总线
try:
from app.event_bus import get_event_bus
get_event_bus().stop()
logger.info("[EventBus] Redis事件总线已停止")
except Exception:
pass
# 优雅关闭:停止所有 Evolution Pipeline
try:
from app.services.evolution_service import EvolutionWatcherManager
EvolutionWatcherManager().shutdown_all()
logger.info("[Evolution] 所有Pipeline已停止")
except Exception:
pass
logger.info("Qihu-AIFlow 正在关闭...")
app = FastAPI(
title="Qihu-AIFlow - 智能自动化PaaS平台",
version="1.0.0",
lifespan=lifespan,
)
# API调用追踪中间件
app.add_middleware(APITrackingMiddleware)
# CORS配置
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# 前台 API
app.include_router(auth.router, prefix="/app/api/auth", tags=["认证"])
app.include_router(chat.router, prefix="/app/api/chat", tags=["AI聊天"])
app.include_router(knowledge.router, prefix="/app/api/knowledge", tags=["知识库"])
app.include_router(market.router, prefix="/app/api/market", tags=["应用市场"])
app.include_router(token_usage.router, prefix="/app/api/token", tags=["Token驾驶舱"])
app.include_router(payment.router, prefix="/app/api/payment", tags=["付费中心"])
app.include_router(profile.router, prefix="/app/api/profile", tags=["个人中心"])
app.include_router(notifications.router, prefix="/app/api/notifications", tags=["消息通知"])
app.include_router(assets.router, prefix="/app/api/assets", tags=["资产管理"])
app.include_router(evolution.router, prefix="/app/api/evolution", tags=["AI自进化"])
# 后台 API
app.include_router(dashboard.router, prefix="/app/api/admin/dashboard", tags=["后台-仪表盘"])
app.include_router(users.router, prefix="/app/api/admin/users", tags=["后台-用户管理"])
app.include_router(models.router, prefix="/app/api/admin/models", tags=["后台-模型管理"])
app.include_router(settings.router, prefix="/app/api/admin/settings", tags=["后台-网站设置"])
app.include_router(payment_admin.router, prefix="/app/api/admin/payment", tags=["后台-支付配置"])
app.include_router(services.router, prefix="/app/api/admin/services", tags=["后台-增值服务"])
app.include_router(membership_admin.router, prefix="/app/api/admin/membership", tags=["后台-会员管理"])
app.include_router(ads.router, prefix="/app/api/admin/ads", tags=["后台-广告管理"])
app.include_router(market_admin.router, prefix="/app/api/admin/market", tags=["后台-应用审核"])
app.include_router(logs.router, prefix="/app/api/admin/logs", tags=["后台-日志风控"])
app.include_router(skills_admin.router, prefix="/app/api/admin/skills", tags=["后台-技能管理"])
app.include_router(agents_admin.router, prefix="/app/api/admin/agents", tags=["后台-Agent管理"])
app.include_router(knowledge_admin.router, prefix="/app/api/admin/knowledge", tags=["后台-知识库管理"])
app.include_router(workflow_admin.router, prefix="/app/api/admin/workflows", tags=["后台-工作流编排"])
app.include_router(asset_admin.router, prefix="/app/api/admin/assets", tags=["后台-资产管理"])
app.include_router(storage_admin.router, prefix="/app/api/admin/storage", tags=["后台-存储配置"])
app.include_router(automation_admin.router, prefix="/app/api/admin/automation", tags=["后台-自动化任务"])
app.include_router(mcp_admin.router, prefix="/app/api/admin/mcp", tags=["后台-MCP服务"])
app.include_router(developer_admin.router, prefix="/app/api/admin/developer", tags=["后台-开发者API"])
app.include_router(developer_tools_router, prefix="/app/api/admin/developer-tools", tags=["dev-tools"])
app.include_router(apps_admin.router, prefix="/app/api/admin/apps", tags=["admin-ai-apps"])
app.include_router(mcp_market_router, prefix="/app/api/admin/mcp-market", tags=["mcp-market"])
app.include_router(code_exec_admin.router, prefix="/app/api/admin/code-exec", tags=["code-exec"])
app.include_router(distro_admin.router, prefix="/app/api/admin/distributions", tags=["distributions"])
app.include_router(privacy_admin.router, prefix="/app/api/admin/privacy", tags=["privacy"])
app.include_router(im_bot_admin.router, prefix="/app/api/admin/im-bot", tags=["im-bot"])
app.include_router(im_bot_user_router, prefix="/app/api/im/wechat", tags=["im-wechat-user"])
app.include_router(im_bot_platform_router, prefix="/app/api/im/platform", tags=["im-platform-user"])
app.include_router(promotion_admin.router, prefix="/app/api/admin/promotion", tags=["promotion"])
app.include_router(tenant_admin.router, prefix="/app/api/admin/tenant", tags=["tenant"])
app.include_router(billing_admin.router, prefix="/app/api/admin/billing", tags=["billing"])
app.include_router(analytics_admin.router, prefix="/app/api/admin/analytics", tags=["analytics"])
app.include_router(observability_admin.router, prefix="/app/api/admin/observability", tags=["后台-调用可观测"])
app.include_router(user_portal.router, prefix="/app/api/portal", tags=["portal"])
app.include_router(achievement_admin.router, prefix="/app/api/admin/achievement", tags=["achievement"])
app.include_router(casdoor_router, prefix="/app/api/auth", tags=["Casdoor SSO"])
@app.get("/app/api/health")
async def health_check():
return {"status": "ok", "service": "Qihu-AIFlow"}