Phase 3.3 多 Agent 编排图引擎: orchestration JSON 列 + 拓扑并行引擎 + local/external/reduce 节点 + 工具沙箱继承 + 后台编排 API

VERSION

@@ -1 +1 @@
-v2026.07.19.6
+v2026.07.19.8

backend/python/app/api/market.py

@@ -1,3 +1,4 @@
+from app.core.credential import encrypt_secret
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy.orm import Session
 from sqlalchemy import and_, desc, asc, func
@@ -557,7 +558,7 @@ def upload_app(
         is_member_free=req.is_member_free if req.is_member_free is not None else False,
         join_promotion=req.join_promotion if req.join_promotion is not None else False,
         agent_endpoint=req.agent_endpoint,
-        agent_api_key=req.api_key,
+        agent_api_key=encrypt_secret(req.api_key),
         status="published",  # 用户创建的应用直接发布
         created_at=datetime.now(),
         updated_at=datetime.now(),

backend/python/app/core/credential.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+"""Phase 3.2 - Agent 凭据加解密 (Fernet 对称加密)。
+
+主密钥获取策略 (防竞态):
+- 优先环境变量 AGENT_MASTER_KEY。
+- 否则读写 /app/secrets/agent_master_key:
+  * 部署阶段由 patch 显式生成并落盘 (原子写), 容器重启/提交后稳定存在。
+  * 运行时只读, 文件存在即读取, 绝不重新生成 —— 避免多进程/多请求并发
+    重建密钥导致新旧密文无法互解。
+  * 仅当文件确实不存在时, 才在文件锁保护下生成 (双重检查, 原子替换)。
+- 存储格式: 密文以 "enc:" 前缀标识; 非 "enc:" 视为旧明文 (向后兼容, 解密原样返回)。
+"""
+import os
+import logging
+import fcntl
+from cryptography.fernet import Fernet
+
+logger = logging.getLogger(__name__)
+
+_MASTER_KEY_FILE = "/app/secrets/agent_master_key"
+_fernet_instance = None
+
+
+def _generate_and_persist_key() -> bytes:
+    key = Fernet.generate_key()
+    os.makedirs(os.path.dirname(_MASTER_KEY_FILE), exist_ok=True)
+    tmp = _MASTER_KEY_FILE + ".tmp"
+    with open(tmp, "w") as f:
+        f.write(key.decode())
+        f.flush()
+        os.fsync(f.fileno())
+    try:
+        os.chmod(tmp, 0o600)
+    except Exception:
+        pass
+    os.replace(tmp, _MASTER_KEY_FILE)  # 原子替换, 避免读到半截文件
+    return key
+
+
+def get_master_key() -> bytes:
+    env_key = os.environ.get("AGENT_MASTER_KEY")
+    if env_key:
+        return env_key.encode() if isinstance(env_key, str) else env_key
+    # 运行时只读: 文件已存在则直接读取 (部署阶段已生成)
+    if os.path.exists(_MASTER_KEY_FILE):
+        try:
+            with open(_MASTER_KEY_FILE, "r") as f:
+                k = f.read().strip()
+                if k:
+                    return k.encode()
+        except Exception as e:
+            logger.warning("[credential] 读取主密钥失败: %s" % e)
+    # 文件缺失: 加锁串行生成, 防多进程并发重建
+    lockf = open(_MASTER_KEY_FILE + ".lock", "w")
+    try:
+        fcntl.flock(lockf, fcntl.LOCK_EX)
+        if os.path.exists(_MASTER_KEY_FILE):  # 双重检查
+            with open(_MASTER_KEY_FILE, "r") as f:
+                k = f.read().strip()
+                if k:
+                    return k.encode()
+        return _generate_and_persist_key()
+    finally:
+        fcntl.flock(lockf, fcntl.LOCK_UN)
+        try:
+            lockf.close()
+        except Exception:
+            pass
+
+
+def _fernet() -> Fernet:
+    global _fernet_instance
+    if _fernet_instance is None:
+        _fernet_instance = Fernet(get_master_key())
+    return _fernet_instance
+
+
+def encrypt_secret(plain):
+    if plain is None or plain == "":
+        return None
+    if isinstance(plain, str) and plain.startswith("enc:"):
+        return plain  # 已是密文, 幂等
+    try:
+        return "enc:" + _fernet().encrypt(plain.encode()).decode()
+    except Exception as e:
+        logger.error("[credential] 加密失败: %s" % e)
+        return None
+
+
+def decrypt_secret(cipher):
+    if cipher is None or cipher == "":
+        return None
+    if isinstance(cipher, str) and cipher.startswith("enc:"):
+        try:
+            return _fernet().decrypt(cipher[4:].encode()).decode()
+        except Exception as e:
+            logger.error("[credential] 解密失败(密钥不匹配或数据损坏): %s" % e)
+            return None
+    return cipher  # 旧明文, 向后兼容
+
+
+def mask_secret(plain):
+    if not plain:
+        return ""
+    return ("*" * 8) + plain[-4:]

backend/python/app/core/orchestrator.py

@@ -0,0 +1,395 @@
+"""app/core/orchestrator.py — Phase 3.3 多 Agent 编排图引擎。
+
+设计目标：
+- 以轻量 DAG 描述多 Agent 协作：节点(llm/local/external/reduce) + 有向边(数据流)。
+- 拓扑分层后按层并行执行(asyncio.gather)，互相独立的节点真正并发。
+- 失败隔离：每个节点异常不影响其它分支；per-node on_error=halt|continue。
+- 节点执行复用既有原语(_build_system_prompt / _execute_skill_tool / _call_external_agent /
+  3.1 工具沙箱)，完全向后兼容——agent.orchestration 为 null 时 chat_service 走原逻辑。
+- 并发安全：每个节点用独立 SessionLocal 会话做 DB 读，避免共享请求会话跨协程竞态。
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from typing import Any, Optional
+
+import httpx
+
+logger = logging.getLogger("aiflow.orchestrator")
+
+VALID_NODE_TYPES = ("llm", "local", "external", "reduce")
+
+
+class OrchestrationError(ValueError):
+    """图结构非法或执行被 halt。"""
+
+
+# ---------------------------------------------------------------------------
+# 校验 + 拓扑
+# ---------------------------------------------------------------------------
+def validate_graph(graph: Any) -> None:
+    if not isinstance(graph, dict):
+        raise OrchestrationError("orchestration 必须为对象")
+    nodes = graph.get("nodes")
+    edges = graph.get("edges")
+    if not isinstance(nodes, list) or not nodes:
+        raise OrchestrationError("nodes 必须为非空数组")
+    if not isinstance(edges, list):
+        raise OrchestrationError("edges 必须为数组")
+
+    ids = set()
+    for i, n in enumerate(nodes):
+        if not isinstance(n, dict):
+            raise OrchestrationError(f"节点 #{i} 必须为对象")
+        nid = n.get("id")
+        if not nid or not isinstance(nid, str):
+            raise OrchestrationError(f"节点 #{i} 缺少字符串 id")
+        if nid in ids:
+            raise OrchestrationError(f"节点 id 重复: {nid}")
+        ids.add(nid)
+        ntype = n.get("type")
+        if ntype not in VALID_NODE_TYPES:
+            raise OrchestrationError(
+                f"节点 {nid} 类型非法: {ntype} (允许 {', '.join(VALID_NODE_TYPES)})"
+            )
+        if ntype in ("local", "external") and not n.get("agent_id"):
+            raise OrchestrationError(f"节点 {nid} 类型 {ntype} 必须指定 agent_id")
+
+    for i, e in enumerate(edges):
+        if not isinstance(e, dict):
+            raise OrchestrationError(f"边 #{i} 必须为对象")
+        f = e.get("from")
+        t = e.get("to")
+        if f not in ids:
+            raise OrchestrationError(f"边 #{i} 的 from={f} 指向不存在的节点")
+        if t not in ids:
+            raise OrchestrationError(f"边 #{i} 的 to={t} 指向不存在的节点")
+        if f == t:
+            raise OrchestrationError(f"边 #{i} 存在自环: {f}")
+
+
+def topological_layers(graph: dict) -> list:
+    """Kahn 算法返回分层(每层内节点可并行)；含环则抛错。"""
+    nodes = graph["nodes"]
+    edges = graph.get("edges", [])
+    indeg = {n["id"]: 0 for n in nodes}
+    adj = {n["id"]: [] for n in nodes}
+    for e in edges:
+        f, t = e["from"], e["to"]
+        adj[f].append(t)
+        indeg[t] += 1
+    order = {n["id"]: i for i, n in enumerate(nodes)}
+    layers: list = []
+    remaining = set(indeg)
+    while remaining:
+        layer = sorted([nid for nid in remaining if indeg[nid] == 0], key=lambda x: order[x])
+        if not layer:
+            cycle = sorted(remaining, key=lambda x: order[x])
+            raise OrchestrationError(f"检测到环路，无法拓扑排序: {cycle}")
+        layers.append(layer)
+        for nid in layer:
+            remaining.discard(nid)
+            for nxt in adj[nid]:
+                indeg[nxt] -= 1
+    return layers
+
+
+def _node_by_id(graph: dict, nid: str) -> dict:
+    for n in graph["nodes"]:
+        if n["id"] == nid:
+            return n
+    return {}
+
+
+def _predecessors(graph: dict, nid: str) -> list:
+    return [e["from"] for e in graph.get("edges", []) if e["to"] == nid]
+
+
+def _successors(graph: dict, nid: str) -> list:
+    return [e["to"] for e in graph.get("edges", []) if e["from"] == nid]
+
+
+def _find_output_node(graph: dict) -> str:
+    if graph.get("output_node"):
+        return graph["output_node"]
+    sinks = [n["id"] for n in graph["nodes"] if not _successors(graph, n["id"])]
+    if len(sinks) == 1:
+        return sinks[0]
+    for s in sinks:
+        if _node_by_id(graph, s).get("type") == "reduce":
+            return s
+    if sinks:
+        return sinks[0]
+    return graph["nodes"][-1]["id"]
+
+
+def _assemble_node_input(query: str, upstream: str, is_entry: bool) -> str:
+    """入口节点直接吃用户原问题；下游节点综合原问题 + 上游输出。"""
+    if is_entry:
+        return query or ""
+    if upstream:
+        return f"原始用户问题:\n{query}\n\n上游节点输出:\n{upstream}"
+    return query or ""
+
+
+# ---------------------------------------------------------------------------
+# 模型解析 + LLM 调用原语
+# ---------------------------------------------------------------------------
+async def _resolve_model_and_endpoint(model_id, db):
+    from app.models import AIModel, ModelProvider
+    from ..capability_registry import resolve_auto_model
+
+    ai_model = None
+    if model_id and model_id != "auto":
+        ai_model = db.query(AIModel).filter(AIModel.id == model_id).first()
+    if not ai_model:
+        ai_model = resolve_auto_model("text", db)
+    if not ai_model:
+        raise OrchestrationError("无法解析到可用模型")
+    provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first()
+    api_key = ai_model.api_key or (provider.api_key if provider else None)
+    api_url = ai_model.endpoint
+    if not api_url and provider and provider.api_base_url:
+        api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions"
+    if not api_url or not api_key:
+        raise OrchestrationError(f"节点模型 {ai_model.name} 缺少 endpoint 或 api_key")
+    return ai_model, api_url, api_key
+
+
+async def _llm_complete(system_prompt: str, user_content: str, model_id, db, user_id) -> str:
+    from app.api.chat_utils import _proxy_for_url
+    from app.core.observability import span
+
+    ai_model, api_url, api_key = await _resolve_model_and_endpoint(model_id, db)
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_content},
+    ]
+    request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048}
+    async with span("orchestrator.llm", model=ai_model.id):
+        async with httpx.AsyncClient(
+            timeout=ai_model.timeout or 120,
+            proxy=_proxy_for_url(api_url),
+            trust_env=False,
+        ) as client:
+            resp = await client.post(
+                api_url,
+                headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+                json=request_body,
+            )
+        if resp.status_code != 200:
+            raise OrchestrationError(f"LLM 调用失败 HTTP {resp.status_code}: {resp.text[:200]}")
+        data = resp.json()
+        return data["choices"][0]["message"].get("content") or ""
+
+
+# ---------------------------------------------------------------------------
+# local 节点：复用单 Agent system prompt + 受限 tool round + 3.1 沙箱
+# ---------------------------------------------------------------------------
+async def _run_local_agent_turn(agent: dict, node_input: str, db, user_id, conv_id) -> dict:
+    from app.api.chat_skill import _resolve_market_skills, _execute_skill_tool
+    from app.api.chat_context import _build_system_prompt, _retrieve_kb_context
+    from app.api.chat_service import _resolve_agent_tool_policy, _check_tool_policy
+    from app.api.chat_utils import _proxy_for_url
+
+    skills = _resolve_market_skills(db, agent.get("skill_ids") or [], user_id)
+    kb_context = _retrieve_kb_context(db, agent.get("knowledge_base_ids") or [], node_input)
+    system_prompt, function_definitions = _build_system_prompt(
+        skills=skills,
+        agents=[agent],
+        kb_context=kb_context,
+        user_content=node_input,
+        auto_discovered=False,
+    )
+    ai_model, api_url, api_key = await _resolve_model_and_endpoint(agent.get("model_id"), db)
+    tool_policy = _resolve_agent_tool_policy([agent])
+
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": node_input},
+    ]
+    called_skills: list = []
+    tool_call_count = 0
+    max_tool_rounds = 5
+    last_content = ""
+    async with httpx.AsyncClient(
+        timeout=ai_model.timeout or 120,
+        proxy=_proxy_for_url(api_url),
+        trust_env=False,
+    ) as client:
+        for round_num in range(max_tool_rounds):
+            request_body = {"model": ai_model.name, "messages": messages, "max_tokens": 2048}
+            if function_definitions:
+                request_body["tools"] = [
+                    {"type": "function", "function": fd} for fd in function_definitions
+                ]
+                request_body["tool_choice"] = "auto"
+            resp = await client.post(
+                api_url,
+                headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+                json=request_body,
+            )
+            if resp.status_code != 200:
+                raise OrchestrationError(f"local 节点 LLM 失败 HTTP {resp.status_code}: {resp.text[:200]}")
+            data = resp.json()
+            message = data["choices"][0]["message"]
+            tool_calls = message.get("tool_calls") or []
+            content = message.get("content") or ""
+            last_content = content
+            if not tool_calls or not function_definitions:
+                return {"text": content, "used_tools": called_skills}
+            messages.append({"role": "assistant", "content": content, "tool_calls": tool_calls})
+            round_tool_count = 0
+            for tc in tool_calls:
+                func_name = tc.get("function", {}).get("name", "")
+                check = _check_tool_policy(
+                    func_name, None, skills, tool_policy, tool_call_count, round_tool_count
+                )
+                if not check["allowed"]:
+                    messages.append({
+                        "role": "tool",
+                        "tool_call_id": tc.get("id", f"call_{round_num}"),
+                        "content": f"[工具调用已被策略拦截] {check.get('message', check.get('reason'))}",
+                    })
+                    tool_call_count += 1
+                    round_tool_count += 1
+                    continue
+                tool_call_count += 1
+                round_tool_count += 1
+                try:
+                    func_args = json.loads(tc.get("function", {}).get("arguments", "{}") or "{}")
+                except Exception:
+                    func_args = {}
+                res = _execute_skill_tool(db, func_name, func_args, skills, node_input)
+                called_skills.append(func_name)
+                messages.append({
+                    "role": "tool",
+                    "tool_call_id": tc.get("id", f"call_{round_num}"),
+                    "content": res,
+                })
+    return {"text": last_content, "used_tools": called_skills}
+
+
+# ---------------------------------------------------------------------------
+# 单节点执行（含独立 DB 会话，保证并发安全）
+# ---------------------------------------------------------------------------
+async def _run_node(node, graph, blackboard, query, user_id, conv_id, trace_id) -> dict:
+    from app.api.chat_skill import _resolve_market_agents
+    from app.database import SessionLocal
+
+    nid = node["id"]
+    ntype = node["type"]
+    preds = _predecessors(graph, nid)
+    pred_results = [blackboard[p] for p in preds]
+    is_entry = not preds
+
+    if pred_results:
+        parts = []
+        for b in pred_results:
+            label = f"节点 {b.get('node_id')}/{b.get('node_type')}"
+            if b.get("ok"):
+                parts.append(f"【{label}】\n{b.get('output', '')}")
+            else:
+                parts.append(f"【{label} 执行失败】\n错误: {b.get('error', '')}")
+        upstream = "\n\n".join(parts)
+    else:
+        upstream = ""
+
+    with SessionLocal() as db:
+        if ntype in ("llm", "reduce"):
+            if ntype == "llm":
+                sys_prompt = node.get("system_prompt") or (
+                    "你是一个处理节点，请根据上游输入完成指定的文本变换、摘要或路由判断，"
+                    "只输出处理后的结果，不要多余解释。"
+                )
+            else:
+                sys_prompt = node.get("system_prompt") or (
+                    "你是一个结果汇总器，请将多个子任务的输出综合成连贯、完整、面向用户的最终回答。"
+                )
+            uc = _assemble_node_input(query, upstream, is_entry)
+            text = await _llm_complete(sys_prompt, uc, node.get("model_id"), db, user_id)
+            return {"ok": True, "node_id": nid, "node_type": ntype, "output": text}
+
+        if ntype == "local":
+            agents = _resolve_market_agents(db, [node["agent_id"]], user_id)
+            if not agents:
+                raise OrchestrationError(f"local 节点 {nid} 引用的 agent_id={node['agent_id']} 不存在")
+            ni = _assemble_node_input(query, upstream, is_entry)
+            res = await _run_local_agent_turn(agents[0], ni, db, user_id, conv_id)
+            return {
+                "ok": True,
+                "node_id": nid,
+                "node_type": "local",
+                "output": res["text"],
+                "used_tools": res.get("used_tools"),
+            }
+
+        if ntype == "external":
+            from app.api.chat_agents import _call_external_agent
+
+            agents = _resolve_market_agents(db, [node["agent_id"]], user_id)
+            if not agents:
+                raise OrchestrationError(f"external 节点 {nid} 引用的 agent_id={node['agent_id']} 不存在")
+            ni = _assemble_node_input(query, upstream, is_entry)
+            answer, _ = await _call_external_agent(
+                db, agents[0], query=ni, user_id=user_id, ext_conversation_id=None, history=[]
+            )
+            return {"ok": True, "node_id": nid, "node_type": "external", "output": answer or ""}
+
+    raise OrchestrationError(f"未知节点类型 {ntype}")
+
+
+# ---------------------------------------------------------------------------
+# 入口
+# ---------------------------------------------------------------------------
+async def run_graph(graph, query, db, user_id, conv_id, trace_id=None):
+    """执行编排图，返回 (answer: str, trace: dict)。
+
+    - 拓扑分层后逐层 asyncio.gather 并行执行。
+    - 节点异常按 on_error=halt|continue 处理（默认 continue，失败时下游拿到错误作为输入）。
+    """
+    validate_graph(graph)
+    layers = topological_layers(graph)
+    blackboard: dict = {}
+    trace = {"nodes": [], "trace_id": trace_id, "layers": layers}
+
+    for layer in layers:
+        tasks = [
+            _run_node(_node_by_id(graph, nid), graph, blackboard, query, user_id, conv_id, trace_id)
+            for nid in layer
+        ]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        for nid, res in zip(layer, results):
+            node = _node_by_id(graph, nid)
+            if isinstance(res, Exception):
+                on_error = node.get("on_error", "continue")
+                err_msg = str(res)
+                blackboard[nid] = {
+                    "ok": False,
+                    "node_id": nid,
+                    "node_type": node["type"],
+                    "error": err_msg,
+                }
+                trace["nodes"].append({"id": nid, "type": node["type"], "ok": False, "error": err_msg})
+                logger.warning(f"[orchestrator] 节点 {nid} 失败 on_error={on_error}: {err_msg}")
+                if on_error == "halt":
+                    raise OrchestrationError(f"节点 {nid} 执行失败(halt): {err_msg}")
+            else:
+                blackboard[nid] = res
+                trace["nodes"].append({
+                    "id": nid,
+                    "type": res.get("node_type"),
+                    "ok": True,
+                    "used_tools": res.get("used_tools"),
+                })
+
+    output_node = _find_output_node(graph)
+    final = blackboard.get(output_node, {})
+    if final.get("ok"):
+        answer = final.get("output", "")
+    else:
+        answer = f"⚠️ 编排输出节点 {output_node} 执行失败: {final.get('error', '未知错误')}"
+    trace["output_node"] = output_node
+    return answer, trace

migrations/023_widen_agent_api_key.sql

@@ -0,0 +1,11 @@
+-- Phase 3.2: 放宽 market_apps.agent_api_key 容量以容纳 Fernet 密文
+SET @db = DATABASE();
+SET @tbl = 'market_apps';
+SET @col = 'agent_api_key';
+SELECT COLUMN_TYPE INTO @ctype FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col;
+SET @sql = IF(@ctype IS NULL OR @ctype NOT LIKE '%text%',
+  CONCAT('ALTER TABLE ', @tbl, ' MODIFY COLUMN ', @col, ' TEXT NULL COMMENT \'Agent 接入 API Key(3.2 起 Fernet 加密存储)\''),
+  'SELECT 1');
+PREPARE stmt FROM @sql;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;

migrations/024_add_orchestration.sql

@@ -0,0 +1,15 @@
+-- Phase 3.3: MarketApp 增加 orchestration（Agent 多 Agent 编排图，JSON）
+-- 幂等：仅当列不存在时才 ALTER
+SET @db = 'aiflow';
+SET @tbl = 'market_apps';
+SET @col = 'orchestration';
+SET @exists = (
+    SELECT COUNT(*) FROM information_schema.columns
+    WHERE table_schema = @db AND table_name = @tbl AND column_name = @col
+);
+SET @sql = IF(@exists = 0,
+    "ALTER TABLE market_apps ADD COLUMN orchestration JSON NULL COMMENT 'Agent 多Agent编排图(3.3)'",
+    'SELECT 1');
+PREPARE stmt FROM @sql;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;