feat(chat): support AGNES image/video models in AI chat - backend _generate_media (images/generations + async videos polling) and frontend MarkdownContent media rendering + model capability badges; nginx proxy_read_timeout raised to 600s for long media generation
This commit is contained in:
qyyfw committed
1 parent
774f1648dc
commit
d62848ee98
3 files changed
+267
-25
No files matched your search
@@ -12,6 +12,7 @@ import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
import re
|
||||
import json
|
||||
import asyncio
|
||||
import math
|
||||
import httpx
|
||||
import os
|
||||
@@ -40,6 +41,108 @@ def cst_now():
|
||||
return datetime.now(CST_TZ)
|
||||
|
||||
|
||||
def _html_escape(s: str) -> str:
|
||||
return (str(s).replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
.replace('"', """).replace("'", "'"))
|
||||
|
||||
|
||||
async def _generate_media(db, ai_model, prompt: str, user_id: str = None) -> str:
|
||||
"""图像/视频生成模型统一入口：按 capabilities 调用 AGNES 等媒体生成 API。
|
||||
返回可直接在 AI 聊天中渲染的富文本(含 <img>/<video> 标签)。"""
|
||||
provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first()
|
||||
api_key = ai_model.api_key or (provider.api_key if provider else None)
|
||||
if not api_key:
|
||||
return "⚠️ 该媒体模型缺少 API Key，请在后台「大模型库」中配置。"
|
||||
|
||||
caps = [c.lower() for c in (ai_model.capabilities or [])]
|
||||
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||||
timeout = ai_model.timeout or 120
|
||||
|
||||
# ============ 图像生成 ============
|
||||
if "image" in caps:
|
||||
api_url = ai_model.endpoint
|
||||
if not api_url:
|
||||
api_url = f"{provider.api_base_url.rstrip('/')}/v1/images/generations" if provider else ""
|
||||
if not api_url:
|
||||
return "⚠️ 该图像模型未配置 Endpoint。"
|
||||
payload = {
|
||||
"model": ai_model.name,
|
||||
"prompt": prompt,
|
||||
"size": "1024x768",
|
||||
"extra_body": {"response_format": "url"},
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
resp = await client.post(api_url, headers=headers, json=payload)
|
||||
if resp.status_code != 200:
|
||||
return f"⚠️ 图像生成失败 (HTTP {resp.status_code}): {resp.text[:200]}"
|
||||
data = resp.json()
|
||||
items = data.get("data") or []
|
||||
if not items:
|
||||
return f"⚠️ 图像生成失败：响应无图片数据 {str(data)[:200]}"
|
||||
item = items[0]
|
||||
url = item.get("url") or item.get("b64_json")
|
||||
if not url:
|
||||
return f"⚠️ 图像生成失败：未返回图片地址 {str(data)[:200]}"
|
||||
if not url.startswith("http"):
|
||||
url = f"data:image/png;base64,{url}"
|
||||
return (f'<img class="md-img" src="{url}" alt="生成图片"/>\n\n'
|
||||
f'**提示词：** {_html_escape(prompt)}')
|
||||
except Exception as e:
|
||||
return f"⚠️ 图像生成异常：{str(e)[:200]}"
|
||||
|
||||
# ============ 视频生成 (异步任务 + 轮询) ============
|
||||
if "video" in caps:
|
||||
api_url = ai_model.endpoint
|
||||
if not api_url:
|
||||
api_url = f"{provider.api_base_url.rstrip('/')}/v1/videos" if provider else ""
|
||||
if not api_url:
|
||||
return "⚠️ 该视频模型未配置 Endpoint。"
|
||||
base = api_url.split("/v1")[0] if "/v1" in api_url else api_url
|
||||
payload = {
|
||||
"model": ai_model.name,
|
||||
"prompt": prompt,
|
||||
"height": 768,
|
||||
"width": 1152,
|
||||
"num_frames": 121,
|
||||
"frame_rate": 24,
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
resp = await client.post(api_url, headers=headers, json=payload)
|
||||
if resp.status_code != 200:
|
||||
return f"⚠️ 视频生成失败 (HTTP {resp.status_code}): {resp.text[:200]}"
|
||||
data = resp.json()
|
||||
video_id = data.get("video_id") or data.get("task_id") or data.get("id")
|
||||
if not video_id:
|
||||
return f"⚠️ 视频生成失败：未返回任务ID {str(data)[:200]}"
|
||||
# 轮询结果(最多约 5 分钟)
|
||||
result_url = None
|
||||
err_msg = None
|
||||
for _ in range(60):
|
||||
await asyncio.sleep(5)
|
||||
r2 = await client.get(f"{base}/agnesapi?video_id={video_id}", headers=headers)
|
||||
if r2.status_code != 200:
|
||||
continue
|
||||
rd = r2.json()
|
||||
st = rd.get("status")
|
||||
if st == "completed":
|
||||
result_url = rd.get("url")
|
||||
break
|
||||
if st == "failed":
|
||||
err_msg = rd.get("error")
|
||||
break
|
||||
if result_url:
|
||||
return (f'<video class="md-video" controls src="{result_url}"></video>\n\n'
|
||||
f'**提示词：** {_html_escape(prompt)}')
|
||||
return f"⚠️ 视频生成失败：{err_msg or '生成超时，请稍后在对话中重试'}"
|
||||
except Exception as e:
|
||||
return f"⚠️ 视频生成异常：{str(e)[:200]}"
|
||||
|
||||
return "⚠️ 未知的媒体模型类型。"
|
||||
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Request Models
|
||||
# ============================================================
|
||||
@@ -881,7 +984,14 @@ async def send_message(
|
||||
if model_id:
|
||||
ai_model = db.query(AIModel).filter(AIModel.id == model_id).first()
|
||||
|
||||
if ai_model and ai_model.endpoint:
|
||||
# ====== 图像/视频生成模型分支 ======
|
||||
caps = [c.lower() for c in (ai_model.capabilities or [])] if ai_model else []
|
||||
if ai_model and ("image" in caps or "video" in caps):
|
||||
try:
|
||||
ai_content = await _generate_media(db, ai_model, req.content, current_user.id)
|
||||
except Exception as e:
|
||||
ai_content = f"⚠️ 媒体生成失败：{str(e)[:200]}"
|
||||
elif ai_model and ai_model.endpoint:
|
||||
provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first()
|
||||
api_key = ai_model.api_key or (provider.api_key if provider else None)
|
||||
api_url = ai_model.endpoint
|
||||
@@ -1333,6 +1443,7 @@ def get_models(db: Session = Depends(get_db)):
|
||||
"display_name": getattr(m, "display_name", None) or m.name,
|
||||
"provider_id": m.provider_id,
|
||||
"provider_name": provider.name if provider else "",
|
||||
"capabilities": m.capabilities or [],
|
||||
"status": m.status,
|
||||
"price_per_million_tokens": float(m.price_per_million_tokens or 0),
|
||||
"is_enabled": m.is_enabled,
|
||||
|
||||
@@ -385,9 +385,46 @@
|
||||
|
||||
/* Markdown Rendering */
|
||||
.markdown-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
display: block;
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
.md-p { margin: 0 0 8px; }
|
||||
.md-p:last-child { margin-bottom: 0; }
|
||||
.md-blockquote {
|
||||
border-left: 3px solid #6C63FF;
|
||||
padding: 4px 12px;
|
||||
margin: 8px 0;
|
||||
background: rgba(0,0,0,0.04);
|
||||
border-radius: 0 4px 4px 0;
|
||||
color: #5A6480;
|
||||
}
|
||||
.md-link {
|
||||
color: #6C63FF;
|
||||
text-decoration: underline;
|
||||
}
|
||||
.md-table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
margin: 8px 0;
|
||||
font-size: 13px;
|
||||
}
|
||||
.md-td {
|
||||
border: 1px solid rgba(0,0,0,0.1);
|
||||
padding: 6px 10px;
|
||||
text-align: left;
|
||||
}
|
||||
.md-tr:first-child .md-td {
|
||||
font-weight: 600;
|
||||
background: rgba(0,0,0,0.03);
|
||||
}
|
||||
.md-ul {
|
||||
margin: 4px 0 8px;
|
||||
padding-left: 1.5em;
|
||||
}
|
||||
.md-ul .md-li {
|
||||
margin: 2px 0;
|
||||
list-style: disc;
|
||||
}
|
||||
|
||||
.message-bubble.user .markdown-body {
|
||||
@@ -630,6 +667,20 @@
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* 模型能力徽标（图像/视频） */
|
||||
.model-cap-badge {
|
||||
margin-left: auto;
|
||||
flex-shrink: 0;
|
||||
font-size: 11px;
|
||||
line-height: 1;
|
||||
padding: 3px 7px;
|
||||
border-radius: 999px;
|
||||
background: linear-gradient(135deg, #6C63FF 0%, #8B7CFF 100%);
|
||||
color: #fff;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.3px;
|
||||
}
|
||||
|
||||
.dropdown-item.danger {
|
||||
color: var(--color-error);
|
||||
}
|
||||
@@ -1045,3 +1096,28 @@
|
||||
border-color: transparent;
|
||||
transform: none;
|
||||
}
|
||||
|
||||
/* AI 聊天中生成的图像 / 视频 */
|
||||
.md-img {
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
max-height: 420px;
|
||||
width: auto;
|
||||
height: auto;
|
||||
border-radius: var(--radius-md);
|
||||
margin: var(--space-sm) 0;
|
||||
box-shadow: var(--shadow-md);
|
||||
background: #fff;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.md-video {
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
width: 100%;
|
||||
max-height: 460px;
|
||||
border-radius: var(--radius-md);
|
||||
margin: var(--space-sm) 0;
|
||||
box-shadow: var(--shadow-md);
|
||||
background: #000;
|
||||
}
|
||||
@@ -4,40 +4,86 @@ import { chatAPI, knowledgeAPI, marketAPI, evolutionAPI } from '@/api';
|
||||
import type { Conversation, Message } from '@/types';
|
||||
import './ChatPage.css';
|
||||
|
||||
// ===== Simple Markdown Renderer =====
|
||||
// ===== Enhanced Markdown Renderer =====
|
||||
const MarkdownContent: React.FC<{ content: string }> = ({ content }) => {
|
||||
const html = useMemo(() => {
|
||||
let text = content;
|
||||
// Code blocks (```)
|
||||
text = text.replace(/```(\w*)\n([\s\S]*?)```/g, (_: string, lang: string, code: string) => {
|
||||
return `<pre class="md-code-block"><code class="md-code">${escapeHtml(code.trim())}</code></pre>`;
|
||||
|
||||
// 1. Extract code blocks first (protect from other replacements)
|
||||
const codeBlocks: string[] = [];
|
||||
text = text.replace(/```(\w*)\n?([\s\S]*?)```/g, (_m: string, _lang: string, code: string) => {
|
||||
const idx = codeBlocks.length;
|
||||
codeBlocks.push('<pre class="md-code-block"><code class="md-code">' + escapeHtml(code.trim()) + '</code></pre>');
|
||||
return '\x00CODEBLOCK' + idx + '\x00';
|
||||
});
|
||||
// Inline code (`)
|
||||
text = text.replace(/`([^`]+)`/g, '<code class="md-inline-code">$1</code>');
|
||||
// Bold (**text**)
|
||||
text = text.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>');
|
||||
// Italic (*text*)
|
||||
text = text.replace(/\*(.+?)\*/g, '<em>$1</em>');
|
||||
// Headers (## heading)
|
||||
|
||||
// 2. Extract inline code
|
||||
const inlineCodes: string[] = [];
|
||||
text = text.replace(/`([^`]+)`/g, (_m: string, code: string) => {
|
||||
const idx = inlineCodes.length;
|
||||
inlineCodes.push('<code class="md-inline-code">' + escapeHtml(code) + '</code>');
|
||||
return '\x00INLINE' + idx + '\x00';
|
||||
});
|
||||
|
||||
// 2.5 Extract media blocks (<img>/<video>) to protect from other replacements
|
||||
const mediaBlocks: string[] = [];
|
||||
text = text.replace(/<(img|video)[^>]*\/?>/gi, (_m: string) => {
|
||||
const idx = mediaBlocks.length;
|
||||
mediaBlocks.push(_m);
|
||||
return '\x00MEDIA' + idx + '\x00';
|
||||
});
|
||||
|
||||
// 3. Headers
|
||||
text = text.replace(/^#### (.+)$/gm, '<h4 class="md-h4">$1</h4>');
|
||||
text = text.replace(/^### (.+)$/gm, '<h4 class="md-h4">$1</h4>');
|
||||
text = text.replace(/^## (.+)$/gm, '<h3 class="md-h3">$1</h3>');
|
||||
text = text.replace(/^# (.+)$/gm, '<h2 class="md-h2">$1</h2>');
|
||||
// Unordered list items
|
||||
text = text.replace(/^- (.+)$/gm, '<li class="md-li">$1</li>');
|
||||
// Ordered list items
|
||||
text = text.replace(/^\d+\. (.+)$/gm, '<li class="md-li">$1</li>');
|
||||
// Horizontal rule
|
||||
text = text.replace(/^---$/gm, '<hr class="md-hr">');
|
||||
// Paragraph spacing: double newlines
|
||||
text = text.replace(/\n\n/g, '<br/><br/>');
|
||||
// Single newlines to <br>
|
||||
|
||||
// 4. Horizontal rule
|
||||
text = text.replace(/^---+$/gm, '<hr class="md-hr">');
|
||||
|
||||
// 5. Blockquotes
|
||||
text = text.replace(/^> (.+)$/gm, '<blockquote class="md-blockquote">$1</blockquote>');
|
||||
|
||||
// 6. Links [text](url)
|
||||
text = text.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a class="md-link" href="$2" target="_blank" rel="noopener">$1</a>');
|
||||
|
||||
// 7. Bold and italic
|
||||
text = text.replace(/\*\*\*(.+?)\*\*\*/g, '<strong><em>$1</em></strong>');
|
||||
text = text.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>');
|
||||
text = text.replace(/(?<!\w)\*(?!\*)(.+?)(?<!\*)\*(?!\w)/g, '<em>$1</em>');
|
||||
|
||||
// 8. Lists
|
||||
text = text.replace(/^\s*[-*] (.+)$/gm, '<li class="md-li">$1</li>');
|
||||
text = text.replace(/^\s*\d+\. (.+)$/gm, '<li class="md-li">$1</li>');
|
||||
// Wrap consecutive <li> in <ul>
|
||||
text = text.replace(/((?:<li class="md-li">[\s\S]*?<\/li>\n?)+)/g, '<ul class="md-ul">$1</ul>');
|
||||
|
||||
// 9. Convert double newlines to paragraph breaks, single to <br/>
|
||||
text = text.replace(/\n\n/g, '</p><p class="md-p">');
|
||||
text = text.replace(/\n/g, '<br/>');
|
||||
|
||||
// 10. Restore code blocks and inline codes
|
||||
text = text.replace(/\x00CODEBLOCK(\d+)\x00/g, (_m: string, idx: string) => codeBlocks[parseInt(idx)] || '');
|
||||
text = text.replace(/\x00INLINE(\d+)\x00/g, (_m: string, idx: string) => inlineCodes[parseInt(idx)] || '');
|
||||
text = text.replace(/\x00MEDIA(\d+)\x00/g, (_m: string, idx: string) => mediaBlocks[parseInt(idx)] || '');
|
||||
|
||||
// 11. Wrap in paragraph and clean up
|
||||
text = '<p class="md-p">' + text + '</p>';
|
||||
text = text.replace(/<p class="md-p"><\/p>/g, '');
|
||||
text = text.replace(/<p class="md-p">(<h[234][\s\S]*?<\/h[234]>)<\/p>/g, '$1');
|
||||
text = text.replace(/<p class="md-p">(<pre[\s\S]*?<\/pre>)<\/p>/g, '$1');
|
||||
text = text.replace(/<p class="md-p">(<ul[\s\S]*?<\/ul>)<\/p>/g, '$1');
|
||||
text = text.replace(/<p class="md-p">(<hr[\s\S]*?)<\/p>/g, '$1');
|
||||
text = text.replace(/<p class="md-p">(<blockquote[\s\S]*?<\/blockquote>)<\/p>/g, '$1');
|
||||
|
||||
return text;
|
||||
}, [content]);
|
||||
|
||||
return <div className="markdown-body" dangerouslySetInnerHTML={{ __html: html }} />;
|
||||
};
|
||||
|
||||
|
||||
function escapeHtml(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, '&')
|
||||
@@ -547,8 +593,10 @@ const ChatPage: React.FC = () => {
|
||||
kbName: res.data.knowledge_base_name,
|
||||
kbId: res.data.knowledge_base_id,
|
||||
});
|
||||
// 使用后端返回的真实计数（幂等，始终为 1），避免前端重复累加导致计数虚高
|
||||
const newCount = res.data.knowledge_graph_count ?? 1;
|
||||
setConversations(prev =>
|
||||
prev.map(c => c.id === convId ? { ...c, knowledgeGraphCount: (c.knowledgeGraphCount || 0) + 1 } : c)
|
||||
prev.map(c => c.id === convId ? { ...c, knowledgeGraphCount: newCount } : c)
|
||||
);
|
||||
} else {
|
||||
setKgError(res.message || '转换失败');
|
||||
@@ -753,6 +801,13 @@ const ChatPage: React.FC = () => {
|
||||
<div className="model-dropdown-item">
|
||||
<span>{model.name}</span>
|
||||
<span className="model-provider-tag">{model.provider_name || model.providerName || ''}</span>
|
||||
{(() => {
|
||||
const caps = model.capabilities || [];
|
||||
const label = caps.includes('image') && caps.includes('video') ? '图像/视频'
|
||||
: caps.includes('image') ? '图像'
|
||||
: caps.includes('video') ? '视频' : '';
|
||||
return label ? <span className="model-cap-badge">{label}</span> : null;
|
||||
})()}
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
|
||||
Reference in new issue
Block a user