T4: AnySearch 联网检索接入 (anysearch_client trust_env直连 NO_PROXY, kb_router use_web RRF融合, chat_context 自动注入联网上下文, admin 支持 anysearch 服务类型, 新增 POST /api/knowledge/web-search 端点)

backend/python/app/anysearch_client.py

@@ -0,0 +1,118 @@
+"""
+AnySearch 联网检索客户端
+======================
+调用 AnySearch 网页/领域搜索 API (https://api.anysearch.com/v1/search)，
+为 AI 聊天注入实时联网检索结果，亦可被后端端点 / Agent 工具复用。
+
+约束：平台容器存在智能路由代理，对外部 API 偶发重置；本客户端使用
+trust_env=False 直连 (等价于 NO_PROXY=api.anysearch.com)，规避代理重置。
+失败一律返回空/降级结果，保证聊天链路不中断。
+"""
+import logging
+from typing import List, Dict, Optional
+
+logger = logging.getLogger("aiflow.anysearch")
+
+DEFAULT_API_URL = "https://api.anysearch.com/v1/search"
+DEFAULT_ZONE = "cn"
+REQUEST_TIMEOUT = 8.0
+
+
+async def anysearch_search(
+    api_key: str,
+    api_url: str = DEFAULT_API_URL,
+    query: str = "",
+    top_k: int = 5,
+    zone: str = DEFAULT_ZONE,
+    freshness: Optional[str] = None,
+    timeout: float = REQUEST_TIMEOUT,
+) -> List[Dict]:
+    """调用 AnySearch 搜索，返回结构化结果列表。
+
+    每个结果:
+        {
+            "content": snippet 摘要文本,
+            "score": 0.9..0.5 递减,
+            "source": "web_anysearch",
+            "metadata": {"url", "title", "source_domain", "zone"}
+        }
+    失败时返回 [] (不抛异常)。
+    """
+    if not api_key:
+        return []
+    try:
+        import httpx
+    except ImportError:
+        logger.warning("httpx not available for AnySearch")
+        return []
+
+    payload = {
+        "query": query,
+        "zone": zone,
+        "top_k": max(1, min(int(top_k), 20)),
+    }
+    if freshness:
+        payload["freshness"] = freshness
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {api_key}",
+    }
+    try:
+        # trust_env=False -> 忽略容器内 HTTP(S)_PROXY，直连 api.anysearch.com
+        async with httpx.AsyncClient(trust_env=False, timeout=timeout) as client:
+            resp = await client.post(api_url, json=payload, headers=headers)
+        if resp.status_code != 200:
+            logger.warning(f"AnySearch HTTP {resp.status_code}: {resp.text[:200]}")
+            return []
+        data = resp.json()
+        if data.get("code") != 0:
+            logger.warning(f"AnySearch biz error: {data.get('message')}")
+            return []
+        results = (data.get("data") or {}).get("results") or []
+        hits = []
+        for idx, r in enumerate(results[:top_k]):
+            title = r.get("title", "") or ""
+            url = r.get("url", "") or ""
+            snippet = r.get("snippet") or r.get("content") or ""
+            if not snippet and not title:
+                continue
+            try:
+                from urllib.parse import urlparse
+                domain = urlparse(url).netloc
+            except Exception:
+                domain = ""
+            hits.append({
+                "content": snippet,
+                "score": max(0.4, 0.9 - idx * 0.08),
+                "source": "web_anysearch",
+                "metadata": {
+                    "url": url,
+                    "title": title,
+                    "source_domain": domain,
+                    "zone": zone,
+                },
+            })
+        return hits
+    except Exception as e:
+        logger.warning(f"AnySearch search failed: {e}")
+        return []
+
+
+async def anysearch_health(api_key: str, api_url: str = DEFAULT_API_URL, timeout: float = REQUEST_TIMEOUT) -> Dict:
+    """连通性探测：返回 {"status": "connected"/"disconnected", "detail": ...}。"""
+    if not api_key:
+        return {"status": "disconnected", "detail": "未配置 API Key", "connected": False, "usable": False}
+    try:
+        import httpx
+        async with httpx.AsyncClient(trust_env=False, timeout=timeout) as client:
+            resp = await client.post(
+                api_url,
+                json={"query": "ping", "zone": DEFAULT_ZONE, "top_k": 1},
+                headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
+            )
+        if resp.status_code == 200:
+            return {"status": "connected", "detail": "AnySearch 连接正常", "connected": True, "usable": True}
+        return {"status": "error", "detail": f"HTTP {resp.status_code}", "connected": False, "usable": False}
+    except Exception as e:
+        return {"status": "error", "detail": str(e)[:200], "connected": False, "usable": False}

backend/python/app/kb_router.py

@@ -1,204 +1,262 @@
-"""
+"""
-知识库路由层
+知识库路由层
-============
+============
-将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。
+将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。
-根据知识库配置自动选择最优检索策略。
+根据知识库配置自动选择最优检索策略。
-
+
-使用方式:
+使用方式:
-    from app.kb_router import kb_router_search
+    from app.kb_router import kb_router_search
-    results = await kb_router_search(query, kb_id, top_k=5)
+    results = await kb_router_search(query, kb_id, top_k=5)
-"""
+"""
-
+
-import logging
+import logging
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional
-
+
-logger = logging.getLogger("aiflow.kb_router")
+logger = logging.getLogger("aiflow.kb_router")
-
+
-
+
-async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]:
+async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]:
-    """获取已配置的外部KB服务（仅 dify）"""
+    """获取已配置的外部KB服务（仅 dify）"""
-    from sqlalchemy import text
+    from sqlalchemy import text
-    try:
+    try:
-        query = "SELECT * FROM external_kb_services"
+        query = "SELECT * FROM external_kb_services"
-        if enabled_only:
+        if enabled_only:
-            query += " WHERE is_enabled = 1"
+            query += " WHERE is_enabled = 1"
-        query += " ORDER BY created_at DESC"
+        query += " ORDER BY created_at DESC"
-        result = db_session.execute(text(query))
+        result = db_session.execute(text(query))
-        columns = result.keys()
+        columns = result.keys()
-        return [dict(zip(columns, row)) for row in result.fetchall()]
+        return [dict(zip(columns, row)) for row in result.fetchall()]
-    except Exception as e:
+    except Exception as e:
-        logger.error(f"Failed to get external services: {e}")
+        logger.error(f"Failed to get external services: {e}")
-        return []
+        return []
-
+
-
+
-async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
+async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
-    """检索 Dify 知识库（走 app.dify_kb 客户端）"""
+    """检索 Dify 知识库（走 app.dify_kb 客户端）"""
-    try:
+    try:
-        from app.dify_kb import dify_retrieve
+        from app.dify_kb import dify_retrieve
-        api_url = service.get("api_url", "")
+        api_url = service.get("api_url", "")
-        api_key = service.get("api_key", "")
+        api_key = service.get("api_key", "")
-        dataset_name = None
+        dataset_name = None
-        cfg = service.get("config")
+        cfg = service.get("config")
-        if isinstance(cfg, str):
+        if isinstance(cfg, str):
-            try:
+            try:
-                import json
+                import json
-                cfg = json.loads(cfg)
+                cfg = json.loads(cfg)
-            except Exception:
+            except Exception:
-                cfg = None
+                cfg = None
-        if isinstance(cfg, dict):
+        if isinstance(cfg, dict):
-            dataset_name = cfg.get("dataset_name")
+            dataset_name = cfg.get("dataset_name")
-        hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
+        hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
-        return hits
+        return hits
-    except Exception as e:
+    except Exception as e:
-        logger.error(f"Dify search error: {e}")
+        logger.error(f"Dify search error: {e}")
-        return []
+        return []
-
+
-
+
-async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
+async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
-    """路由到对应的外部KB服务（当前仅支持 dify）"""
+    """路由到对应的外部KB服务（当前仅支持 dify）"""
-    service_type = service.get("service_type", "")
+    service_type = service.get("service_type", "")
-    if service_type == "dify":
+    if service_type == "dify":
-        return await _search_dify(service, query, top_k)
+        return await _search_dify(service, query, top_k)
-    else:
+    else:
-        logger.warning(f"Unknown external KB type: {service_type}")
+        logger.warning(f"Unknown external KB type: {service_type}")
-        return []
+        return []
-
+
-
+async def _get_web_services(db_session, enabled_only: bool = True) -> List[Dict]:
-async def kb_router_search(
+    """获取已启用的 AnySearch 联网检索服务配置。"""
-    query: str,
+    from sqlalchemy import text
-    kb_id: Optional[str] = None,
+    try:
-    kb_ids: Optional[List[str]] = None,
+        query = "SELECT * FROM external_kb_services WHERE service_type='anysearch'"
-    top_k: int = 5,
+        if enabled_only:
-    bm25_results: Optional[List[Dict]] = None,
+            query += " AND is_enabled = 1"
-    db_session=None,
+        query += " ORDER BY created_at DESC"
-    use_vector: bool = True,
+        result = db_session.execute(text(query))
-    use_external: bool = True,
+        columns = result.keys()
-    use_tree: bool = True,
+        return [dict(zip(columns, row)) for row in result.fetchall()]
-) -> List[Dict]:
+    except Exception as e:
-    """
+        logger.error(f"Failed to get AnySearch services: {e}")
-    知识库路由层 — 混合检索
+        return []
-
+
-    检索策略:
+
-    1. 本地 BM25 检索 (MySQL, 已有)
+async def _search_anysearch(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
-    2. 向量语义检索 (PostgreSQL+pgvector, 新增)
+    """调用 AnySearch 联网检索（走 app.anysearch_client）。"""
-    3. 结构化分片段检索 (TreeSearch, 新增: 知识库目录树 + chunk 级精准命中)
+    try:
-    4. 外部KB检索 (Dify, 按配置)
+        from app.anysearch_client import anysearch_search
-
+        api_key = service.get("api_key", "")
-    融合方式: Reciprocal Rank Fusion (RRF)
+        api_url = service.get("api_url") or "https://api.anysearch.com/v1/search"
-
+        cfg = service.get("config")
-    Args:
+        if isinstance(cfg, str):
-        query: 查询文本
+            try:
-        kb_id: 知识库ID
+                import json
-        top_k: 返回结果数
+                cfg = json.loads(cfg)
-        bm25_results: BM25检索结果
+            except Exception:
-        db_session: 数据库会话(用于获取外部KB配置与分片段检索)
+                cfg = None
-        use_vector: 是否使用向量检索
+        cfg = cfg or {}
-        use_external: 是否使用外部KB检索
+        zone = cfg.get("zone", "cn")
-        use_tree: 是否使用 TreeSearch 结构化分片段检索
+        freshness = cfg.get("freshness")
-
+        k = int(cfg.get("top_k", top_k))
-    Returns:
+        return await anysearch_search(api_key, api_url, query, top_k=k, zone=zone, freshness=freshness)
-        融合后的检索结果列表
+    except Exception as e:
-    """
+        logger.error(f"AnySearch search error: {e}")
-    all_results = []
+        return []
-
+
-    # 1. BM25 结果
+
-    if bm25_results:
+async def kb_router_search(
-        for rank, item in enumerate(bm25_results):
+    query: str,
-            all_results.append({
+    kb_id: Optional[str] = None,
-                "content": item.get("content", item.get("text", "")),
+    kb_ids: Optional[List[str]] = None,
-                "score": item.get("score", 1.0 / (60 + rank + 1)),
+    top_k: int = 5,
-                "source": "local_bm25",
+    bm25_results: Optional[List[Dict]] = None,
-                "rank": rank + 1,
+    db_session=None,
-                "metadata": item.get("metadata", {}),
+    use_vector: bool = True,
-            })
+    use_external: bool = True,
-
+    use_tree: bool = True,
-    # 2. 向量检索
+    use_web: bool = False,
-    if use_vector:
+) -> List[Dict]:
-        try:
+    """
-            from app.pgvector_search import vector_search
+    知识库路由层 — 混合检索
-            kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else [None])
+
-            vec_results = []
+    检索策略:
-            for _kid in kb_targets:
+    1. 本地 BM25 检索 (MySQL, 已有)
-                try:
+    2. 向量语义检索 (PostgreSQL+pgvector, 新增)
-                    vec_results.extend(await vector_search(query, _kid, top_k))
+    3. 结构化分片段检索 (TreeSearch, 新增: 知识库目录树 + chunk 级精准命中)
-                except Exception as e:
+    4. 外部KB检索 (Dify, 按配置)
-                    logger.warning(f"Vector search failed for kb={_kid}: {e}")
+    5. 联网检索 (AnySearch, 可选 use_web)
-            for rank, item in enumerate(vec_results):
+
-                all_results.append({
+    融合方式: Reciprocal Rank Fusion (RRF)
-                    "content": item.get("content", ""),
+
-                    "score": item.get("score", 0.5),
+    Args:
-                    "source": "vector_pg",
+        query: 查询文本
-                    "rank": rank + 1,
+        kb_id: 知识库ID
-                    "metadata": item.get("metadata", {}),
+        top_k: 返回结果数
-                })
+        bm25_results: BM25检索结果
-        except ImportError:
+        db_session: 数据库会话(用于获取外部KB配置与分片段检索)
-            logger.debug("pgvector_search not available, skipping vector search")
+        use_vector: 是否使用向量检索
-        except Exception as e:
+        use_external: 是否使用外部KB检索
-            logger.warning(f"Vector search failed: {e}")
+        use_tree: 是否使用 TreeSearch 结构化分片段检索
-
+        use_web: 是否使用 AnySearch 联网检索
-    # 2.5 结构化分片段检索 (TreeSearch, Level 2)
+
-    if use_tree and db_session is not None:
+    Returns:
-        try:
+        融合后的检索结果列表
-            from app.tree_search import tree_search
+    """
-            kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else None)
+    all_results = []
-            tree_hits = await tree_search(
+
-                db_session, query, kb_ids=kb_targets, top_k=top_k, mode="hybrid",
+    # 1. BM25 结果
-            )
+    if bm25_results:
-            for rank, item in enumerate(tree_hits):
+        for rank, item in enumerate(bm25_results):
-                all_results.append({
+            all_results.append({
-                    "content": item.get("content", ""),
+                "content": item.get("content", item.get("text", "")),
-                    "score": item.get("score", 0.5),
+                "score": item.get("score", 1.0 / (60 + rank + 1)),
-                    "source": "tree_chunk",
+                "source": "local_bm25",
-                    "rank": rank + 1,
+                "rank": rank + 1,
-                    "metadata": item.get("metadata", {}),
+                "metadata": item.get("metadata", {}),
-                })
+            })
-        except Exception as e:
+
-            logger.warning(f"TreeSearch failed: {e}")
+    # 2. 向量检索
-
+    if use_vector:
-    # 3. 外部KB检索 (Dify)
+        try:
-    if use_external and db_session:
+            from app.pgvector_search import vector_search
-        services = await _get_external_services(db_session)
+            kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else [None])
-        for service in services:
+            vec_results = []
-            if service.get("status") == "connected":
+            for _kid in kb_targets:
-                ext_results = await _search_external(service, query, top_k)
+                try:
-                for rank, item in enumerate(ext_results):
+                    vec_results.extend(await vector_search(query, _kid, top_k))
-                    all_results.append({
+                except Exception as e:
-                        "content": item.get("content", ""),
+                    logger.warning(f"Vector search failed for kb={_kid}: {e}")
-                        "score": item.get("score", 0.5),
+            for rank, item in enumerate(vec_results):
-                        "source": item.get("source", "external_dify"),
+                all_results.append({
-                        "rank": rank + 1,
+                    "content": item.get("content", ""),
-                        "metadata": item.get("metadata", {}),
+                    "score": item.get("score", 0.5),
-                    })
+                    "source": "vector_pg",
-
+                    "rank": rank + 1,
-    if not all_results:
+                    "metadata": item.get("metadata", {}),
-        return []
+                })
-
+        except ImportError:
-    # RRF 融合
+            logger.debug("pgvector_search not available, skipping vector search")
-    rrf_k = 60
+        except Exception as e:
-    score_map = {}
+            logger.warning(f"Vector search failed: {e}")
-    for item in all_results:
+
-        content_key = item["content"][:200]  # 用内容前200字符作为去重键
+    # 2.5 结构化分片段检索 (TreeSearch, Level 2)
-        rrf_score = 1.0 / (rrf_k + item.get("rank", 1))
+    if use_tree and db_session is not None:
-
+        try:
-        if content_key in score_map:
+            from app.tree_search import tree_search
-            score_map[content_key]["score"] += rrf_score
+            kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else None)
-            score_map[content_key]["sources"].append(item["source"])
+            tree_hits = await tree_search(
-        else:
+                db_session, query, kb_ids=kb_targets, top_k=top_k, mode="hybrid",
-            score_map[content_key] = {
+            )
-                "content": item["content"],
+            for rank, item in enumerate(tree_hits):
-                "score": rrf_score,
+                all_results.append({
-                "sources": [item["source"]],
+                    "content": item.get("content", ""),
-                "metadata": item.get("metadata", {}),
+                    "score": item.get("score", 0.5),
-            }
+                    "source": "tree_chunk",
-
+                    "rank": rank + 1,
-    # 按融合分数排序
+                    "metadata": item.get("metadata", {}),
-    merged = sorted(score_map.values(), key=lambda x: x["score"], reverse=True)
+                })
-
+        except Exception as e:
-    # 标准化分数到 0-1
+            logger.warning(f"TreeSearch failed: {e}")
-    if merged:
+
-        max_score = merged[0]["score"]
+    # 3. 外部KB检索 (Dify)
-        for item in merged:
+    if use_external and db_session:
-            item["score"] = item["score"] / max_score if max_score > 0 else 0
+        services = await _get_external_services(db_session)
-            item["sources"] = list(set(item["sources"]))
+        for service in services:
-
+            if service.get("status") == "connected":
-    return merged[:top_k]
+                ext_results = await _search_external(service, query, top_k)
+                for rank, item in enumerate(ext_results):
+                    all_results.append({
+                        "content": item.get("content", ""),
+                        "score": item.get("score", 0.5),
+                        "source": item.get("source", "external_dify"),
+                        "rank": rank + 1,
+                        "metadata": item.get("metadata", {}),
+                    })
+
+    # 4. 联网检索 (AnySearch, 可选)
+    if use_web and db_session is not None:
+        try:
+            web_services = await _get_web_services(db_session)
+            for svc in web_services:
+                web_hits = await _search_anysearch(svc, query, top_k)
+                for rank, item in enumerate(web_hits):
+                    all_results.append({
+                        "content": item.get("content", ""),
+                        "score": item.get("score", 0.6),
+                        "source": item.get("source", "web_anysearch"),
+                        "rank": rank + 1,
+                        "metadata": item.get("metadata", {}),
+                    })
+        except Exception as e:
+            logger.warning(f"AnySearch web search failed: {e}")
+
+    if not all_results:
+        return []
+
+    # RRF 融合
+    rrf_k = 60
+    score_map = {}
+    for item in all_results:
+        content_key = item["content"][:200]  # 用内容前200字符作为去重键
+        rrf_score = 1.0 / (rrf_k + item.get("rank", 1))
+
+        if content_key in score_map:
+            score_map[content_key]["score"] += rrf_score
+            score_map[content_key]["sources"].append(item["source"])
+        else:
+            score_map[content_key] = {
+                "content": item["content"],
+                "score": rrf_score,
+                "sources": [item["source"]],
+                "metadata": item.get("metadata", {}),
+            }
+
+    # 按融合分数排序
+    merged = sorted(score_map.values(), key=lambda x: x["score"], reverse=True)
+
+    # 标准化分数到 0-1
+    if merged:
+        max_score = merged[0]["score"]
+        for item in merged:
+            item["score"] = item["score"] / max_score if max_score > 0 else 0
+            item["sources"] = list(set(item["sources"]))
+
+    return merged[:top_k]