chore(align): sync Git with running container as authoritative source
One-time alignment to close drift between the running container (qihu-aiflow / image qihuaiflow:with-deps) and this Git repository. Container hotfixes previously persisted only via `docker commit` are now captured in Git so Git becomes the single source of truth. Backend: - chat_service.py: accumulated fixes incl. EigenFlux README guidance - chat_context.py / chat_media.py: URL/web-repo context injection - api/url_reader.py: NEW SSRF-safe web/README reader (Issue #725) - skills_admin.py & Skills.tsx: 14-scene capability unification - pgvector_search.py, orchestrator.py, platform_knowledge.py, market.py, admin/models.py: drift sync Frontend: - ChatPage.tsx: collapsible AI think/act block (Issue #724) - MarketPage / EvolutionDashboard / TokenDashboard styling sync Migrations: - 018_normalize_marketapp_scene.sql: fixed capability allowlist (prevents capability wipe on restart) Alignment performed 2026-07-23 05:42:47. No deletions; secrets/venv/node_modules/ caches were not touched (tracked-source-only copy).
This commit is contained in:
1 parent
49f7503978
commit
a08ec87b22
19 files changed
+7474
-7196
No files matched your search
@@ -0,0 +1,163 @@
|
||||
"""服务端 URL 读取器(Issue 2 修复)
|
||||
|
||||
当用户输入包含 URL 且明确要求『读取 / 加入 / 安装 / 分析 / 总结』等意图时,
|
||||
服务端抓取该链接的真实内容并作为参考上下文注入对话,使模型能基于页面真实信息
|
||||
作答(例如按 GitHub 仓库 README 的 Quick Start 引导用户完成安装/加入),
|
||||
而不是凭空编造。
|
||||
|
||||
SSRF 防护:仅允许 http/https;解析域名 IP 后阻断内网/回环/链路本地/云元数据地址;
|
||||
限制跳转、响应体大小与超时。
|
||||
"""
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_URL_RE = re.compile(r'https?://[^\s"\)\\\]>]+', re.IGNORECASE)
|
||||
# 触发意图:显式要求读取/加入/安装/分析/总结/打开/访问 等
|
||||
_INTENT_RE = re.compile(
|
||||
r'(read|join|install|fetch|summar|analys|open|visit|access|browse|'
|
||||
r'读取|加入|安装|打开|访问|浏览|总结|分析|抓取|看下这个|查看该|这个网页|这个页面|这个链接|help me join)',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# 常见私有/保留网段(阻断内网扫描与云元数据)
|
||||
_BLOCKED_NETS = [
|
||||
ipaddress.ip_network('10.0.0.0/8'),
|
||||
ipaddress.ip_network('172.16.0.0/12'),
|
||||
ipaddress.ip_network('192.168.0.0/16'),
|
||||
ipaddress.ip_network('127.0.0.0/8'),
|
||||
ipaddress.ip_network('169.254.0.0/16'),
|
||||
ipaddress.ip_network('::1/128'),
|
||||
ipaddress.ip_network('fc00::/7'),
|
||||
ipaddress.ip_network('fe80::/10'),
|
||||
ipaddress.ip_network('0.0.0.0/8'),
|
||||
]
|
||||
|
||||
_MAX_CHARS = 16000
|
||||
_TIMEOUT = 30.0
|
||||
|
||||
|
||||
def _is_blocked_host(host: str) -> bool:
|
||||
host = (host or '').strip().lower()
|
||||
if not host:
|
||||
return True
|
||||
if re.fullmatch(r'(\d{1,3}\.){3}\d{1,3}', host):
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return True
|
||||
return any(ip in net for net in _BLOCKED_NETS)
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None)
|
||||
for info in infos:
|
||||
ipstr = info[4][0]
|
||||
try:
|
||||
ip = ipaddress.ip_address(ipstr)
|
||||
except ValueError:
|
||||
continue
|
||||
if any(ip in net for net in _BLOCKED_NETS):
|
||||
return True
|
||||
except Exception:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def _make_client():
|
||||
try:
|
||||
return httpx.AsyncClient(trust_env=True, follow_redirects=True, timeout=_TIMEOUT)
|
||||
except Exception:
|
||||
return httpx.AsyncClient(trust_env=False, follow_redirects=True, timeout=_TIMEOUT)
|
||||
|
||||
|
||||
async def _github_text(url: str):
|
||||
p = urlparse(url)
|
||||
if p.netloc.lower() not in ('github.com', 'www.github.com'):
|
||||
return None
|
||||
parts = [x for x in p.path.split('/') if x]
|
||||
if len(parts) < 2:
|
||||
return None
|
||||
owner, repo = parts[0], parts[1]
|
||||
if len(parts) >= 4 and parts[2] == 'blob':
|
||||
ref = parts[3]
|
||||
fpath = '/'.join(parts[4:])
|
||||
raw = f'https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{fpath}'
|
||||
return await _fetch_text(raw)
|
||||
api = f'https://api.github.com/repos/{owner}/{repo}/readme'
|
||||
try:
|
||||
async with await _make_client() as c:
|
||||
r = await c.get(api, headers={
|
||||
'Accept': 'application/vnd.github+json',
|
||||
'User-Agent': 'Mozilla/5.0',
|
||||
})
|
||||
if r.status_code == 200:
|
||||
import base64
|
||||
d = r.json()
|
||||
return base64.b64decode(d.get('content', '')).decode('utf-8', 'replace')
|
||||
except Exception as e:
|
||||
logger.warning('github readme fetch failed: %s', e)
|
||||
return None
|
||||
|
||||
|
||||
async def _fetch_text(url: str) -> str:
|
||||
async with await _make_client() as c:
|
||||
r = await c.get(url, headers={'User-Agent': 'Mozilla/5.0'})
|
||||
r.raise_for_status()
|
||||
data = r.content[:2_000_000]
|
||||
ctype = r.headers.get('content-type', '')
|
||||
text = data.decode('utf-8', 'replace')
|
||||
if 'html' in ctype:
|
||||
return _html_to_text(text)
|
||||
return text
|
||||
|
||||
|
||||
def _html_to_text(html: str) -> str:
|
||||
html = re.sub(r'(?is)<(script|style|head|noscript|svg|template)[^>]*>.*?</\1>', ' ', html)
|
||||
html = re.sub(r'(?is)<!--.*?-->', ' ', html)
|
||||
html = re.sub(r'(?is)<[^>]+>', ' ', html)
|
||||
import html as _html
|
||||
text = _html.unescape(html)
|
||||
text = re.sub(r'[ \t]+', ' ', text)
|
||||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
async def fetch_url_context(text: str):
|
||||
"""若有 URL 且命中读取意图,返回 {'url','text'};否则返回 None。"""
|
||||
if not text:
|
||||
return None
|
||||
m = _URL_RE.search(text)
|
||||
if not m:
|
||||
return None
|
||||
url = m.group(0).rstrip('.,);]')
|
||||
if not _INTENT_RE.search(text):
|
||||
return None
|
||||
try:
|
||||
p = urlparse(url)
|
||||
if p.scheme not in ('http', 'https'):
|
||||
return None
|
||||
if _is_blocked_host(p.hostname):
|
||||
logger.warning('blocked host for url fetch: %s', p.hostname)
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
try:
|
||||
if 'github.com' in (p.hostname or '').lower():
|
||||
body = await _github_text(url)
|
||||
if not body:
|
||||
body = await _fetch_text(url)
|
||||
else:
|
||||
body = await _fetch_text(url)
|
||||
except Exception as e:
|
||||
logger.warning('url fetch failed: %s', e)
|
||||
return None
|
||||
|
||||
if not body or not body.strip():
|
||||
return None
|
||||
return {'url': url, 'text': body[:_MAX_CHARS]}
|
||||
Reference in new issue
Block a user