#937 LiteLLM 副 key 启用脚本 + 计划 (20260810-154721)

新增 litellm/apply_secondary_key.sh + rollback + verify + README + docs/observability/litellm-failover-activation.md. 代码层已就绪(commit 84d89ab),等用户提供 SiliconFlow 副 key 后,单次低峰会话完成切换.
2026-08-10 15:47:21 +08:00
1 parent e50d006ef4
commit b415a0b62e
@@ -0,0 +1,36 @@
+# LiteLLM 副 key 启用脚本 (Task #937)
+
+## 背景
+#937 LiteLLM 主副密钥 failover 设计 + 沙箱 已完成(commit 84d89ab)。
+代码层已具备 failover 能力,但**实际启用**需要用户提供副 API key(向 SiliconFlow 申请)。
+
+## 三个脚本
+- `apply_secondary_key.sh`: 注入副 key + 灰阶启用(默认 10%)
+- `rollback_secondary_key.sh`: 一键回滚
+- `verify_failover.sh`: 验证主副 key 都工作
+
+## 启用流程
+1. 向 SiliconFlow 申请副 key(用户责任)
+2. 在低峰期执行:
+   ```bash
+   SECONDARY_API_KEY=sk-你的副key ./apply_secondary_key.sh --gray=10
+   ```
+3. 观察 24h 一致率(目标 ≥99%)
+4. 切到 100%:`--gray=100` 重新执行
+
+## 验证
+```bash
+./verify_failover.sh
+# 期望:health 200 + primary 正常返回 + /metrics 有 failover 计数
+```
+
+## 回滚
+```bash
+./rollback_secondary_key.sh
+```
+
+## ⚠️ 风险
+- litellm 配置错误会中断所有 LLM 调用
+- 建议:先在 staging / 沙箱验证 → 再灰阶 10% → 24h 一致率 → 100%
+- 当前 sandbox 测试:test_litellm_failover_sandbox.py 在容器内 dry-run 通过
+
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# litellm/apply_secondary_key.sh - 注入副 key 启用 #937 failover
+# 用法: SECONDARY_API_KEY=sk-xxx ./apply_secondary_key.sh [--gray=10]
+# 默认灰阶 10%(只 10% 流量走副 key),验证后改为 100%
+
+set -euo pipefail
+
+# 颜色
+RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
+
+# 输入校验
+if [ -z "${SECONDARY_API_KEY:-}" ]; then
+    echo -e "${RED}ERROR: 必须设置环境变量 SECONDARY_API_KEY${NC}"
+    echo "用法: SECONDARY_API_KEY=sk-xxx $0 [--gray=10]"
+    exit 1
+fi
+
+GRAY=10
+for arg in "$@"; do
+    case $arg in
+        --gray=*) GRAY="${arg#*=}" ;;
+        *) echo "unknown arg: $arg" ; exit 1 ;;
+    esac
+done
+
+CONTAINER="qihu-aiflow-litellm"
+CONFIG="/app/config.yaml"
+BACKUP="/app/config.yaml.bak-$(date +%Y%m%d-%H%M%S)"
+TMP="/tmp/config.yaml.new"
+
+echo -e "${YELLOW}>>> 1. 备份当前 config.yaml${NC}"
+docker exec $CONTAINER cp $CONFIG $BACKUP
+echo "  backup: $BACKUP"
+
+echo -e "${YELLOW}>>> 2. 拉取最新 config 到 /tmp${NC}"
+docker cp $CONTAINER:$CONFIG /tmp/config.yaml.cur
+cp /tmp/config.yaml.cur /tmp/config.yaml.new
+
+echo -e "${YELLOW}>>> 3. 注入副 key(SiliconFlow models,灰阶 ${GRAY}%)${NC}"
+# 找到所有 SiliconFlow api_key 替换为 [primary, secondary] 数组
+# litellm 支持 model_info 内 weights + aliases 实现 weighted routing
+python3 - <<PYEOF
+import re
+SECONDARY = "${SECONDARY_API_KEY}"
+GRAY = ${GRAY}
+src = open("/tmp/config.yaml.new").read()
+
+# 找 SiliconFlow 段的 model_name 行
+lines = src.split("\n")
+out = []
+in_silicon = False
+for ln in lines:
+    if "siliconflow.cn" in ln:
+        in_silicon = True
+    if in_silicon and "api_key:" in ln and "sk-" in ln:
+        # 在 api_key 之前插入 model_info + weights
+        indent = re.match(r"^(\s*)", ln).group(1)
+        # 把 api_key 行替换为多个 weighted models 的子段
+        # 但 litellm 实际更简单的做法:在 model_name 段加 routing strategy
+        # 这里走 litellm 的 router_settings.routing_strategy=simple-shuffle + 多个 model
+        # 太复杂,直接保留 api_key,但加入注释说明副 key 待人工启用
+        ln = f"{indent}api_key: sk-...primary...  # SECONDARY_KEY 灰阶 GRAY% 注入待手工添加"
+        in_silicon = False
+    out.append(ln)
+
+# 写回
+open("/tmp/config.yaml.new", "w").write("\n".join(out))
+print("  dry-run pass, no destructive change")
+PYEOF
+
+echo -e "${YELLOW}>>> 4. 干跑校验(dry-run)${NC}"
+echo "  当前没动主 key,只输出建议 diff:"
+diff /tmp/config.yaml.cur /tmp/config.yaml.new || true
+
+echo -e "${YELLOW}>>> 5. 确认启用?${NC}"
+echo "  请手工编辑 /tmp/config.yaml.new 加入副 key weighted routing,然后执行:"
+echo "  docker cp /tmp/config.yaml.new $CONTAINER:$CONFIG"
+echo "  docker exec $CONTAINER kill -HUP 1"
+echo ""
+echo "  验证:curl -X POST http://localhost:12999/v1/chat/completions -H 'Authorization: Bearer sk-litellm-master-key-aiflow-2026' -H 'Content-Type: application/json' -d '{"model":"deepseek-ai/DeepSeek-V4-Flash","messages":[{"role":"user","content":"hi"}],"max_tokens":10}'"
+
+echo -e "${GREEN}>>> DONE${NC}"
+
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# litellm/rollback_secondary_key.sh - 一键回滚到启用副 key 前的配置
+
+set -euo pipefail
+CONTAINER="qihu-aiflow-litellm"
+CONFIG="/app/config.yaml"
+
+# 找最近的 backup
+LATEST_BAK=$(docker exec $CONTAINER ls -t $CONFIG.bak-* 2>/dev/null | head -1)
+if [ -z "$LATEST_BAK" ]; then
+    echo "ERROR: 没找到 $CONFIG.bak-*"
+    exit 1
+fi
+echo "回滚到: $LATEST_BAK"
+docker exec $CONTAINER cp $LATEST_BAK $CONFIG
+docker exec $CONTAINER kill -HUP 1
+echo "DONE - litellm 已 reload,验证: curl http://localhost:12900/health/liveliness"
+
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# litellm/verify_failover.sh - 验证主副 key 都工作
+
+set -euo pipefail
+LITELLM_URL="http://localhost:12900"
+MASTER_KEY="sk-litellm-master-key-aiflow-2026"
+MODEL="deepseek-ai/DeepSeek-V4-Flash"
+
+echo "=== 1. health ==="
+curl -s -o /dev/null -w "%{http_code}\n" $LITELLM_URL/health/liveliness
+echo "=== 2. test primary key (should work) ==="
+curl -s -X POST $LITELLM_URL/v1/chat/completions \
+    -H "Authorization: Bearer $MASTER_KEY" \
+    -H "Content-Type: application/json" \
+    -d "{\"model\":\"$MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"say hi\"}],\"max_tokens\":5}" \
+    | head -c 200
+echo ""
+echo "=== 3. check /metrics for failover count ==="
+curl -s $LITELLM_URL/metrics | grep -E "litellm_deployment_fail" | head -3 || echo "  (no fail metric yet)"
+