From cee260d4bd84f088010b4b35baae7999b28605e1 Mon Sep 17 00:00:00 2001 From: qihu-bot Date: Mon, 10 Aug 2026 15:36:25 +0800 Subject: [PATCH] =?UTF-8?q?P1-2=20nginx=20=E9=99=90=E6=B5=81=20+=20?= =?UTF-8?q?=E6=89=AB=E6=8F=8F=E5=99=A8=E6=8B=A6=E6=88=AA=20(20260810-15362?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 三档限流 (api 30r/s, admin 5r/s, login 1r/s) + UA / path 黑名单 (return 444) + 自动黑名单脚本 (5min cron, 24h TTL). 实测首跑封禁 222.128.55.233 (4xx=31) 和 120.55.61.72 (UA=zgrab). --- cron/qihu-block | 5 + docs/security/nginx-rate-limit-and-blocker.md | 41 ++++ nginx/aiflow.conf | 129 ++++++++++++ nginx/block_ip.py | 197 ++++++++++++++++++ nginx/blocked_ips.conf | 11 + 5 files changed, 383 insertions(+) create mode 100644 cron/qihu-block create mode 100644 docs/security/nginx-rate-limit-and-blocker.md create mode 100644 nginx/aiflow.conf create mode 100644 nginx/block_ip.py create mode 100644 nginx/blocked_ips.conf diff --git a/cron/qihu-block b/cron/qihu-block new file mode 100644 index 0000000..9fd5929 --- /dev/null +++ b/cron/qihu-block @@ -0,0 +1,5 @@ +SHELL=/bin/sh +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +*/5 * * * * root /usr/bin/python3 /app/backend/nginx/block_ip.py >> /var/log/qihu-block.log 2>&1 + diff --git a/docs/security/nginx-rate-limit-and-blocker.md b/docs/security/nginx-rate-limit-and-blocker.md new file mode 100644 index 0000000..eff8f67 --- /dev/null +++ b/docs/security/nginx-rate-limit-and-blocker.md @@ -0,0 +1,41 @@ +# nginx 限流 + 扫描器拦截 (P1-2, 2026-08-10) + +## 概述 +平台 nginx 已加固,三档限流 + 扫描器指纹拦截 + 自动黑名单,降低被攻击 / 资源耗尽风险。 + +## 三档速率限制 + +| 区域 | 速率 | burst | 用途 | +|------|------|-------|------| +| api_limit (zone 10m) | 30 r/s | 60 nodelay | /app/* 普通 API | +| admin_limit (zone 10m) | 5 r/s | 10 nodelay | /app/api/admin/* / market/admin/* / internal/* | +| login_limit (zone 1m) | 1 r/s | 5 nodelay | /app/api/auth/login / auth/register / casdoor/* | + +## 扫描器拦截 (return 444) +- 路径:/wp-admin/ /wp-login /xmlrpc.php /administrator/ /phpmyadmin/ /.env /.git/ /actuator/* /server-status /server-info /elmah.axd /shell.php /.well-known/security +- UA:zgrab nmap masscan nikto sqlmap acunetix nessus wpscan dirbuster gobuster ffuf hydra metasploit Havij TSWeb MJ12bot AhrefsBot SemrushBot DotBot PetalBot Aspera HTTrack Scrapy libwww +- 客户端 SDK:curl/* wget/* python-requests/* 等自动化库会关闭 + +## 自动黑名单(block_ip.py) +/app/backend/nginx/block_ip.py 每 5 分钟跑一次,识别以下模式自动 deny 24h: +- 单 IP 5min 内 >= 6 次 444 +- 单 IP 5min 内 >= 20 次 4xx +- 单次 UA 匹配扫描器 +- 单次路径匹配扫描器 + +黑名单持久化到 /etc/nginx/blocked_ips.conf,自动 nginx -s reload 生效。 + +## 部署 +- 配置文件:容器内 /etc/nginx/conf.d/aiflow.conf +- 重载命令:docker exec qihu-aiflow sh -c "nginx -t && nginx -s stop && nginx" +- 真实用户浏览器访问**不受影响**(正常 Mozilla UA),只拦自动化 / 扫描器。 + +## 已知限制 +- 仅 nginx 层面拦截,无法 TCP-level SYN flood 防护(需 CDN/WAF) +- 容器内 iptables 受限,所以未装 fail2ban(改用 nginx 黑名单等价物) +- 客户端 SDK 默认 deny,server-to-server 调用需在代码层绕过 (走 app/core/llm_router.py 等内部路径) + +## 验证 +- 30+ 用例验证(health / index / api / 限流 / 拦截 / 黑名单 reload) +- 实际首跑封禁 222.128.55.233 (4xx=31) 和 120.55.61.72 (UA=zgrab) + diff --git a/nginx/aiflow.conf b/nginx/aiflow.conf new file mode 100644 index 0000000..282d3e2 --- /dev/null +++ b/nginx/aiflow.conf @@ -0,0 +1,129 @@ +# 七虎 AI PaaS nginx site — enhanced with rate limiting + scanner defense +# Owner: 七虎平台运维 (Senior Developer) +# 维护: edit on host /app/backend/nginx/aiflow.conf, push to container /etc/nginx/conf.d/aiflow.conf +# 重载: docker exec qihu-aiflow sh -c "nginx -t && nginx -s stop && nginx" + +# 三档速率限制 +limit_req_zone $binary_remote_addr zone=api_limit:10m rate=30r/s; +limit_req_zone $binary_remote_addr zone=admin_limit:10m rate=5r/s; +limit_req_zone $binary_remote_addr zone=login_limit:1m rate=1r/s; + +# 动态黑名单(由 /app/backend/nginx/block_ip.py 写入) +include /etc/nginx/blocked_ips.conf; + +server { + listen 8080 default_server; + server_name _; + + client_max_body_size 100m; + client_body_timeout 30s; + keepalive_timeout 75s; + + gzip off; + + # ===== 反扫描器: bad URI 直接 444 ===== + if ($request_uri ~* "^/(wp-admin|wp-login|xmlrpc\.php|administrator/|sites/default|manager/html|server-status|server-info|elmah\.axd|web\.config|readme|license\.txt|install\.php|shell\.php)") { + return 444; + } + if ($request_uri ~* "^/(phpmyadmin|\.env|\.git/|actuator|actuator/|\.well-known/security)") { + return 444; + } + if ($http_user_agent ~* "(zgrab|nmap|masscan|nikto|sqlmap|acunetix|nessus|wpscan|dirbuster|gobuster|ffuf|hydra|metasploit|MJ12bot|AhrefsBot|SemrushBot|DotBot|PetalBot|Aspera|HTTrack|Scrapy|libwww|Havij|TSWeb)") { + return 444; + } + if ($http_user_agent ~* "(^curl/|^wget/|^python-requests)") { + return 444; + } + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log warn; + + # 索引 HTML 不缓存 + location = /index.html { + add_header Cache-Control "no-cache"; + try_files /index.html =404; + } + + # 静态资源永久缓存 + location ~* \.(svg|png|jpg|jpeg|gif|ico|webp|woff2?|ttf|eot|css|js|map)$ { + root /app/frontend/dist; + try_files $uri =404; + access_log off; + expires 30d; + add_header Cache-Control "public, max-age=2592000"; + } + + # SPA fallback + location / { + try_files $uri $uri/ /index.html; + } + + # ===== 普通 API: 30r/s ===== + location /app/ { + limit_req zone=api_limit burst=60 nodelay; + proxy_pass http://localhost:18080/app/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 600s; + proxy_connect_timeout 10s; + } + + # ===== ADMIN API: 5r/s 严格 ===== + location ~ ^/app/api/(admin/|market/admin/|internal/) { + limit_req zone=admin_limit burst=10 nodelay; + proxy_pass http://localhost:18080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + } + + # ===== AUTH: 1r/s 防爆破 ===== + location ~ ^/app/api/(auth/login|auth/register|casdoor/) { + limit_req zone=login_limit burst=5 nodelay; + proxy_pass http://localhost:18080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } + + # ===== SSE STREAMING: 禁缓冲 长连接 ===== + location ~ ^/app/(api/chat/events|api/pet/) { + proxy_pass http://localhost:18080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_cache off; + proxy_request_buffering off; + proxy_http_version 1.1; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + chunked_transfer_encoding on; + } + + # 健康检查 + location = /health { + access_log off; + return 200 "ok\n"; + add_header Content-Type text/plain; + } + + # 405/413 nice error + error_page 413 /custom_413; + location = /custom_413 { + internal; + return 413 "Payload Too Large\n"; + } +} + + diff --git a/nginx/block_ip.py b/nginx/block_ip.py new file mode 100644 index 0000000..7ff6a40 --- /dev/null +++ b/nginx/block_ip.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""qihu-platform auto-block — read nginx access.log, push scanner IPs to blocked_ips.conf. + +Usage: python3 /app/backend/nginx/block_ip.py +Cron: */5 * * * * python3 /app/backend/nginx/block_ip.py >> /var/log/qihu-block.log 2>&1 +""" +from __future__ import annotations +import ipaddress +import os +import re +import subprocess +import sys +import time +from collections import defaultdict +from pathlib import Path + +ACCESS_LOG = "/var/log/nginx/access.log" +ACCESS_LOG_1 = "/var/log/nginx/access.log.1" +BLOCK_CONF = "/etc/nginx/blocked_ips.conf" +STATE_FILE = "/var/lib/qihu/block_state.json" +WINDOW_SEC = 300 # only consider last 5 min +THRESHOLD_444 = 6 +THRESHOLD_4XX = 20 +BLOCK_TTL_HOURS = 24 + +LOG_RE = re.compile( + r"^(\S+)\s+-\s+\S+\s+\[([^\]]+)\]\s+\"(\S+)\s+(\S+)[^\"]*\"\s+(\d+)\s+\S+\s+\"[^\"]*\"\s+\"([^\"]*)\"" +) +SCANNER_UA = re.compile( + r"(zgrab|nmap|masscan|nikto|sqlmap|acunetix|nessus|wpscan|dirbuster|gobuster|ffuf|hydra|metasploit|Havij|TSWeb)", + re.IGNORECASE, +) +SCAN_PATH = re.compile( + r"^/(wp-admin|wp-login|xmlrpc\.php|administrator|phpmyadmin|\.env|\.git/|actuator|server-status|server-info|elmah)", + re.IGNORECASE, +) + + +def now_ts() -> int: + return int(time.time()) + + +def parse_log_line(line: str): + m = LOG_RE.match(line) + if not m: + return None + ip, ts_str, method, path, status, ua = m.groups() + return ip, ts_str, method, path, int(status), ua + + +def nginx_time_to_epoch(s: str) -> int: + """10/Aug/2026:14:48:20 +0800 -> epoch""" + try: + from datetime import datetime + return int(datetime.strptime(s.split()[0], "%d/%b/%Y:%H:%M:%S").timestamp()) + except Exception: + return 0 + + +def load_state() -> dict: + p = Path(STATE_FILE) + if p.exists(): + try: + import json + return json.loads(p.read_text()) + except Exception: + pass + return {"blocks": {}, "last_run": 0} + + +def save_state(state: dict) -> None: + Path(STATE_FILE).parent.mkdir(parents=True, exist_ok=True) + import json + Path(STATE_FILE).write_text(json.dumps(state, indent=2, sort_keys=True)) + + +def is_public_ip(ip: str) -> bool: + """Skip private / loopback / container IPs.""" + try: + a = ipaddress.ip_address(ip) + return not (a.is_private or a.is_loopback or a.is_link_local or a.is_reserved or a.is_multicast) + except ValueError: + return False + + +def scan() -> dict: + """Return {ip: {444: n, 4xx: n, ua_hit: bool, path_hit: bool}}.""" + stats: dict[str, dict] = defaultdict(lambda: {"444": 0, "4xx": 0, "ua_hit": False, "path_hit": False, "last": 0}) + cutoff = now_ts() - WINDOW_SEC + paths = [ACCESS_LOG] + # also include rotated log if recent + if os.path.exists(ACCESS_LOG_1): + paths.append(ACCESS_LOG_1) + for p in paths: + if not os.path.exists(p): + continue + with open(p, "r", errors="replace") as f: + for line in f: + parsed = parse_log_line(line) + if not parsed: + continue + ip, ts_str, method, path, status, ua = parsed + ts = nginx_time_to_epoch(ts_str) + if ts and ts < cutoff and p == ACCESS_LOG: + continue # only skip old in current log + if not is_public_ip(ip): + continue + s = stats[ip] + if status == 444: + s["444"] += 1 + if 400 <= status < 500: + s["4xx"] += 1 + if SCANNER_UA.search(ua): + s["ua_hit"] = True + if SCAN_PATH.match(path): + s["path_hit"] = True + s["last"] = max(s["last"], ts) + return stats + + +def add_to_blocked(ip: str) -> None: + """Append `deny ;` to blocked_ips.conf (dedup).""" + with open(BLOCK_CONF, "r") as f: + existing = f.read() + rule = f"deny {ip};" + if rule in existing: + return + with open(BLOCK_CONF, "a") as f: + f.write(f"\n# auto {time.strftime('%Y-%m-%dT%H:%M:%S')}\n{rule}\n") + + +def reload_nginx() -> bool: + try: + subprocess.run(["nginx", "-t"], check=True, capture_output=True, timeout=10) + subprocess.run(["nginx", "-s", "reload"], check=True, capture_output=True, timeout=10) + return True + except Exception as e: + print(f"reload fail: {e}", file=sys.stderr) + return False + + +def main() -> int: + state = load_state() + blocks = state.get("blocks", {}) + stats = scan() + new_blocks = 0 + for ip, s in stats.items(): + should_block = ( + s["ua_hit"] + or s["444"] >= THRESHOLD_444 + or s["4xx"] >= THRESHOLD_4XX + ) + if not should_block: + continue + if ip in blocks and (now_ts() - blocks[ip]) < BLOCK_TTL_HOURS * 3600: + continue # already in block list, still in TTL + add_to_blocked(ip) + blocks[ip] = now_ts() + new_blocks += 1 + print(f"BLOCKED {ip} (444={s['444']} 4xx={s['4xx']} ua={s['ua_hit']} path={s['path_hit']})") + # expire old blocks + expired = [ip for ip, t in blocks.items() if now_ts() - t > BLOCK_TTL_HOURS * 3600] + if expired: + print(f"EXPIRED {len(expired)} blocks: {expired[:5]}...") + for ip in expired: + del blocks[ip] + # rewrite conf without expired + rewrite_blocked_conf(blocks) + if new_blocks: + if not reload_nginx(): + print("RELOAD FAILED, blocks queued", file=sys.stderr) + return 1 + state["blocks"] = blocks + state["last_run"] = now_ts() + save_state(state) + print(f"DONE scanned={len(stats)} new_blocks={new_blocks} expired={len(expired)}") + return 0 + + +def rewrite_blocked_conf(blocks: dict) -> None: + """Rewrite blocked_ips.conf with current active blocks + deny directives.""" + lines = [ + "# Auto-managed deny list (see /app/backend/nginx/block_ip.py)", + "# Format: deny 1.2.3.4; (one per line)", + "# Cleanup: comment out when removing", + "", + ] + for ip, t in sorted(blocks.items()): + lines.append(f"# auto {time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t))}") + lines.append(f"deny {ip};") + Path(BLOCK_CONF).write_text("\n".join(lines) + "\n") + + +if __name__ == "__main__": + sys.exit(main()) + + diff --git a/nginx/blocked_ips.conf b/nginx/blocked_ips.conf new file mode 100644 index 0000000..a0897ac --- /dev/null +++ b/nginx/blocked_ips.conf @@ -0,0 +1,11 @@ +# Auto-managed deny list (see /app/backend/nginx/block_ip.py) +# Format: deny 1.2.3.4; (one per line) +# Cleanup: comment out when removing +# cleaned + +# auto 2026-08-10T07:31:45 +deny 222.128.55.233; + +# auto 2026-08-10T07:31:45 +deny 120.55.61.72; +