T3: TreeSearch 结构化文档两级检索 (build_kb_tree + tree_search BM25+向量融合, 接入 kb_router 为 tree_chunk 源, 新增 GET /api/knowledge/tree-search)
This commit is contained in:
1 parent
e30e385f11
commit
be7d792dc2
3 files changed
+2124
-1622
No files matched your search
+1515
-1447
File diff suppressed because it is too large.
Load diff
+204
-175
@@ -1,175 +1,204 @@
|
|||||||
"""
|
"""
|
||||||
知识库路由层
|
知识库路由层
|
||||||
============
|
============
|
||||||
将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。
|
将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。
|
||||||
根据知识库配置自动选择最优检索策略。
|
根据知识库配置自动选择最优检索策略。
|
||||||
|
|
||||||
使用方式:
|
使用方式:
|
||||||
from app.kb_router import kb_router_search
|
from app.kb_router import kb_router_search
|
||||||
results = await kb_router_search(query, kb_id, top_k=5)
|
results = await kb_router_search(query, kb_id, top_k=5)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Dict, Optional
|
from typing import List, Dict, Optional
|
||||||
|
|
||||||
logger = logging.getLogger("aiflow.kb_router")
|
logger = logging.getLogger("aiflow.kb_router")
|
||||||
|
|
||||||
|
|
||||||
async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]:
|
async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]:
|
||||||
"""获取已配置的外部KB服务(仅 dify)"""
|
"""获取已配置的外部KB服务(仅 dify)"""
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
try:
|
try:
|
||||||
query = "SELECT * FROM external_kb_services"
|
query = "SELECT * FROM external_kb_services"
|
||||||
if enabled_only:
|
if enabled_only:
|
||||||
query += " WHERE is_enabled = 1"
|
query += " WHERE is_enabled = 1"
|
||||||
query += " ORDER BY created_at DESC"
|
query += " ORDER BY created_at DESC"
|
||||||
result = db_session.execute(text(query))
|
result = db_session.execute(text(query))
|
||||||
columns = result.keys()
|
columns = result.keys()
|
||||||
return [dict(zip(columns, row)) for row in result.fetchall()]
|
return [dict(zip(columns, row)) for row in result.fetchall()]
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to get external services: {e}")
|
logger.error(f"Failed to get external services: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
||||||
"""检索 Dify 知识库(走 app.dify_kb 客户端)"""
|
"""检索 Dify 知识库(走 app.dify_kb 客户端)"""
|
||||||
try:
|
try:
|
||||||
from app.dify_kb import dify_retrieve
|
from app.dify_kb import dify_retrieve
|
||||||
api_url = service.get("api_url", "")
|
api_url = service.get("api_url", "")
|
||||||
api_key = service.get("api_key", "")
|
api_key = service.get("api_key", "")
|
||||||
dataset_name = None
|
dataset_name = None
|
||||||
cfg = service.get("config")
|
cfg = service.get("config")
|
||||||
if isinstance(cfg, str):
|
if isinstance(cfg, str):
|
||||||
try:
|
try:
|
||||||
import json
|
import json
|
||||||
cfg = json.loads(cfg)
|
cfg = json.loads(cfg)
|
||||||
except Exception:
|
except Exception:
|
||||||
cfg = None
|
cfg = None
|
||||||
if isinstance(cfg, dict):
|
if isinstance(cfg, dict):
|
||||||
dataset_name = cfg.get("dataset_name")
|
dataset_name = cfg.get("dataset_name")
|
||||||
hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
|
hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
|
||||||
return hits
|
return hits
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Dify search error: {e}")
|
logger.error(f"Dify search error: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
||||||
"""路由到对应的外部KB服务(当前仅支持 dify)"""
|
"""路由到对应的外部KB服务(当前仅支持 dify)"""
|
||||||
service_type = service.get("service_type", "")
|
service_type = service.get("service_type", "")
|
||||||
if service_type == "dify":
|
if service_type == "dify":
|
||||||
return await _search_dify(service, query, top_k)
|
return await _search_dify(service, query, top_k)
|
||||||
else:
|
else:
|
||||||
logger.warning(f"Unknown external KB type: {service_type}")
|
logger.warning(f"Unknown external KB type: {service_type}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
async def kb_router_search(
|
async def kb_router_search(
|
||||||
query: str,
|
query: str,
|
||||||
kb_id: Optional[str] = None,
|
kb_id: Optional[str] = None,
|
||||||
top_k: int = 5,
|
kb_ids: Optional[List[str]] = None,
|
||||||
bm25_results: Optional[List[Dict]] = None,
|
top_k: int = 5,
|
||||||
db_session=None,
|
bm25_results: Optional[List[Dict]] = None,
|
||||||
use_vector: bool = True,
|
db_session=None,
|
||||||
use_external: bool = True,
|
use_vector: bool = True,
|
||||||
) -> List[Dict]:
|
use_external: bool = True,
|
||||||
"""
|
use_tree: bool = True,
|
||||||
知识库路由层 — 混合检索
|
) -> List[Dict]:
|
||||||
|
"""
|
||||||
检索策略:
|
知识库路由层 — 混合检索
|
||||||
1. 本地 BM25 检索 (MySQL, 已有)
|
|
||||||
2. 向量语义检索 (PostgreSQL+pgvector, 新增)
|
检索策略:
|
||||||
3. 外部KB检索 (Dify, 按配置)
|
1. 本地 BM25 检索 (MySQL, 已有)
|
||||||
|
2. 向量语义检索 (PostgreSQL+pgvector, 新增)
|
||||||
融合方式: Reciprocal Rank Fusion (RRF)
|
3. 结构化分片段检索 (TreeSearch, 新增: 知识库目录树 + chunk 级精准命中)
|
||||||
|
4. 外部KB检索 (Dify, 按配置)
|
||||||
Args:
|
|
||||||
query: 查询文本
|
融合方式: Reciprocal Rank Fusion (RRF)
|
||||||
kb_id: 知识库ID
|
|
||||||
top_k: 返回结果数
|
Args:
|
||||||
bm25_results: BM25检索结果
|
query: 查询文本
|
||||||
db_session: 数据库会话(用于获取外部KB配置)
|
kb_id: 知识库ID
|
||||||
use_vector: 是否使用向量检索
|
top_k: 返回结果数
|
||||||
use_external: 是否使用外部KB检索
|
bm25_results: BM25检索结果
|
||||||
|
db_session: 数据库会话(用于获取外部KB配置与分片段检索)
|
||||||
Returns:
|
use_vector: 是否使用向量检索
|
||||||
融合后的检索结果列表
|
use_external: 是否使用外部KB检索
|
||||||
"""
|
use_tree: 是否使用 TreeSearch 结构化分片段检索
|
||||||
all_results = []
|
|
||||||
|
Returns:
|
||||||
# 1. BM25 结果
|
融合后的检索结果列表
|
||||||
if bm25_results:
|
"""
|
||||||
for rank, item in enumerate(bm25_results):
|
all_results = []
|
||||||
all_results.append({
|
|
||||||
"content": item.get("content", item.get("text", "")),
|
# 1. BM25 结果
|
||||||
"score": item.get("score", 1.0 / (60 + rank + 1)),
|
if bm25_results:
|
||||||
"source": "local_bm25",
|
for rank, item in enumerate(bm25_results):
|
||||||
"rank": rank + 1,
|
all_results.append({
|
||||||
"metadata": item.get("metadata", {}),
|
"content": item.get("content", item.get("text", "")),
|
||||||
})
|
"score": item.get("score", 1.0 / (60 + rank + 1)),
|
||||||
|
"source": "local_bm25",
|
||||||
# 2. 向量检索
|
"rank": rank + 1,
|
||||||
if use_vector:
|
"metadata": item.get("metadata", {}),
|
||||||
try:
|
})
|
||||||
from app.pgvector_search import vector_search
|
|
||||||
vec_results = await vector_search(query, kb_id, top_k)
|
# 2. 向量检索
|
||||||
for rank, item in enumerate(vec_results):
|
if use_vector:
|
||||||
all_results.append({
|
try:
|
||||||
"content": item.get("content", ""),
|
from app.pgvector_search import vector_search
|
||||||
"score": item.get("score", 0.5),
|
kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else [None])
|
||||||
"source": "vector_pg",
|
vec_results = []
|
||||||
"rank": rank + 1,
|
for _kid in kb_targets:
|
||||||
"metadata": item.get("metadata", {}),
|
try:
|
||||||
})
|
vec_results.extend(await vector_search(query, _kid, top_k))
|
||||||
except ImportError:
|
except Exception as e:
|
||||||
logger.debug("pgvector_search not available, skipping vector search")
|
logger.warning(f"Vector search failed for kb={_kid}: {e}")
|
||||||
except Exception as e:
|
for rank, item in enumerate(vec_results):
|
||||||
logger.warning(f"Vector search failed: {e}")
|
all_results.append({
|
||||||
|
"content": item.get("content", ""),
|
||||||
# 3. 外部KB检索 (Dify)
|
"score": item.get("score", 0.5),
|
||||||
if use_external and db_session:
|
"source": "vector_pg",
|
||||||
services = await _get_external_services(db_session)
|
"rank": rank + 1,
|
||||||
for service in services:
|
"metadata": item.get("metadata", {}),
|
||||||
if service.get("status") == "connected":
|
})
|
||||||
ext_results = await _search_external(service, query, top_k)
|
except ImportError:
|
||||||
for rank, item in enumerate(ext_results):
|
logger.debug("pgvector_search not available, skipping vector search")
|
||||||
all_results.append({
|
except Exception as e:
|
||||||
"content": item.get("content", ""),
|
logger.warning(f"Vector search failed: {e}")
|
||||||
"score": item.get("score", 0.5),
|
|
||||||
"source": item.get("source", "external_dify"),
|
# 2.5 结构化分片段检索 (TreeSearch, Level 2)
|
||||||
"rank": rank + 1,
|
if use_tree and db_session is not None:
|
||||||
"metadata": item.get("metadata", {}),
|
try:
|
||||||
})
|
from app.tree_search import tree_search
|
||||||
|
kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else None)
|
||||||
if not all_results:
|
tree_hits = await tree_search(
|
||||||
return []
|
db_session, query, kb_ids=kb_targets, top_k=top_k, mode="hybrid",
|
||||||
|
)
|
||||||
# RRF 融合
|
for rank, item in enumerate(tree_hits):
|
||||||
rrf_k = 60
|
all_results.append({
|
||||||
score_map = {}
|
"content": item.get("content", ""),
|
||||||
for item in all_results:
|
"score": item.get("score", 0.5),
|
||||||
content_key = item["content"][:200] # 用内容前200字符作为去重键
|
"source": "tree_chunk",
|
||||||
rrf_score = 1.0 / (rrf_k + item.get("rank", 1))
|
"rank": rank + 1,
|
||||||
|
"metadata": item.get("metadata", {}),
|
||||||
if content_key in score_map:
|
})
|
||||||
score_map[content_key]["score"] += rrf_score
|
except Exception as e:
|
||||||
score_map[content_key]["sources"].append(item["source"])
|
logger.warning(f"TreeSearch failed: {e}")
|
||||||
else:
|
|
||||||
score_map[content_key] = {
|
# 3. 外部KB检索 (Dify)
|
||||||
"content": item["content"],
|
if use_external and db_session:
|
||||||
"score": rrf_score,
|
services = await _get_external_services(db_session)
|
||||||
"sources": [item["source"]],
|
for service in services:
|
||||||
"metadata": item.get("metadata", {}),
|
if service.get("status") == "connected":
|
||||||
}
|
ext_results = await _search_external(service, query, top_k)
|
||||||
|
for rank, item in enumerate(ext_results):
|
||||||
# 按融合分数排序
|
all_results.append({
|
||||||
merged = sorted(score_map.values(), key=lambda x: x["score"], reverse=True)
|
"content": item.get("content", ""),
|
||||||
|
"score": item.get("score", 0.5),
|
||||||
# 标准化分数到 0-1
|
"source": item.get("source", "external_dify"),
|
||||||
if merged:
|
"rank": rank + 1,
|
||||||
max_score = merged[0]["score"]
|
"metadata": item.get("metadata", {}),
|
||||||
for item in merged:
|
})
|
||||||
item["score"] = item["score"] / max_score if max_score > 0 else 0
|
|
||||||
item["sources"] = list(set(item["sources"]))
|
if not all_results:
|
||||||
|
return []
|
||||||
return merged[:top_k]
|
|
||||||
|
# RRF 融合
|
||||||
|
rrf_k = 60
|
||||||
|
score_map = {}
|
||||||
|
for item in all_results:
|
||||||
|
content_key = item["content"][:200] # 用内容前200字符作为去重键
|
||||||
|
rrf_score = 1.0 / (rrf_k + item.get("rank", 1))
|
||||||
|
|
||||||
|
if content_key in score_map:
|
||||||
|
score_map[content_key]["score"] += rrf_score
|
||||||
|
score_map[content_key]["sources"].append(item["source"])
|
||||||
|
else:
|
||||||
|
score_map[content_key] = {
|
||||||
|
"content": item["content"],
|
||||||
|
"score": rrf_score,
|
||||||
|
"sources": [item["source"]],
|
||||||
|
"metadata": item.get("metadata", {}),
|
||||||
|
}
|
||||||
|
|
||||||
|
# 按融合分数排序
|
||||||
|
merged = sorted(score_map.values(), key=lambda x: x["score"], reverse=True)
|
||||||
|
|
||||||
|
# 标准化分数到 0-1
|
||||||
|
if merged:
|
||||||
|
max_score = merged[0]["score"]
|
||||||
|
for item in merged:
|
||||||
|
item["score"] = item["score"] / max_score if max_score > 0 else 0
|
||||||
|
item["sources"] = list(set(item["sources"]))
|
||||||
|
|
||||||
|
return merged[:top_k]
|
||||||
@@ -0,0 +1,405 @@
|
|||||||
|
"""
|
||||||
|
TreeSearch — 结构化文档两级检索
|
||||||
|
================================
|
||||||
|
|
||||||
|
为知识库提供有结构的「两级检索」能力(零额外基础设施成本,复用 MySQL 已存的分段数据):
|
||||||
|
|
||||||
|
Level 1 (build_kb_tree) : 知识库 -> 目录 -> 文档 的嵌套结构树,用于浏览/定位/面包屑。
|
||||||
|
Level 2 (tree_search) : 在给定作用域(整个 KB / 某目录 / 某文档)内做分段(chunk)级检索,
|
||||||
|
BM25 关键词排序 + 可选 pgvector 语义向量 boost,返回带完整
|
||||||
|
面包屑(KB名 -> 目录路径 -> 文件名 -> 段序号/标题)的精准命中。
|
||||||
|
|
||||||
|
设计要点:
|
||||||
|
* Level 2 直接消费 knowledge_document_chunks 表(content + chunk_metadata.title/keywords),
|
||||||
|
因此召回的是「文档内部的精确片段」,而非整篇文档——这正是 RAG 想要的细粒度上下文。
|
||||||
|
* 与 kb_router_search 融合:tree_search 作为 RRF 的一个新来源 (source="tree_chunk") 接入,
|
||||||
|
聊天 KB 上下文自动获得结构化分片段命中。
|
||||||
|
* 向量 boost 为可选且优雅降级:当 pgvector 无对应数据时,自动退化为纯 BM25,不影响主链路。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import math
|
||||||
|
from typing import List, Dict, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger("aiflow.tree_search")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 中文友好的分词(jieba 优先,否则按字符 bigram + 英文正则兜底)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def _tokenize(text: str) -> List[str]:
|
||||||
|
if not text:
|
||||||
|
return []
|
||||||
|
toks: List[str] = []
|
||||||
|
try:
|
||||||
|
import re
|
||||||
|
# 英文/数字 token
|
||||||
|
for m in re.findall(r"[a-zA-Z0-9_]+", text.lower()):
|
||||||
|
if len(m) >= 1:
|
||||||
|
toks.append(m)
|
||||||
|
# 中文按 jieba 或字符切分
|
||||||
|
cjk = "".join(re.findall(r"[一-鿿]", text))
|
||||||
|
if cjk:
|
||||||
|
try:
|
||||||
|
import jieba
|
||||||
|
for w in jieba.lcut(cjk):
|
||||||
|
w = w.strip().lower()
|
||||||
|
if w:
|
||||||
|
toks.append(w)
|
||||||
|
except Exception:
|
||||||
|
# 字符级 bigram(兼顾召回与精度)
|
||||||
|
for i in range(len(cjk)):
|
||||||
|
toks.append(cjk[i])
|
||||||
|
if i + 1 < len(cjk):
|
||||||
|
toks.append(cjk[i:i + 2])
|
||||||
|
except Exception:
|
||||||
|
toks = [t for t in text.lower().split() if t]
|
||||||
|
return toks
|
||||||
|
|
||||||
|
|
||||||
|
def _chunk_display_text(chunk) -> str:
|
||||||
|
"""拼装分段的检索文本:正文 + 标题 + 关键词(来自 chunk_metadata)。"""
|
||||||
|
parts = []
|
||||||
|
content = getattr(chunk, "content", None) or ""
|
||||||
|
parts.append(content)
|
||||||
|
md = getattr(chunk, "chunk_metadata", None) or {}
|
||||||
|
if isinstance(md, dict):
|
||||||
|
title = md.get("title") or md.get("heading") or ""
|
||||||
|
keywords = md.get("keywords") or []
|
||||||
|
if title:
|
||||||
|
parts.append(title)
|
||||||
|
if isinstance(keywords, list) and keywords:
|
||||||
|
parts.append(" ".join(str(k) for k in keywords))
|
||||||
|
elif isinstance(keywords, str) and keywords:
|
||||||
|
parts.append(keywords)
|
||||||
|
return "\n".join(p for p in parts if p).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _chunk_title(chunk) -> str:
|
||||||
|
md = getattr(chunk, "chunk_metadata", None) or {}
|
||||||
|
if isinstance(md, dict):
|
||||||
|
return md.get("title") or md.get("heading") or ""
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# BM25(在作用域内的分段语料上即时计算 IDF)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def _bm25_rank(query_tokens: List[str], corpus: List[Dict], k1: float = 1.5, b: float = 0.75) -> List[Dict]:
|
||||||
|
"""corpus: [{"id":..., "tokens":[...], "chunk":<orm>, "text":str}] -> 返回带 bm25 分的列表"""
|
||||||
|
if not query_tokens or not corpus:
|
||||||
|
return []
|
||||||
|
n = len(corpus)
|
||||||
|
df = {}
|
||||||
|
for qt in set(query_tokens):
|
||||||
|
df[qt] = sum(1 for d in corpus if qt in d["tokens"])
|
||||||
|
idf = {qt: math.log((n - df[qt] + 0.5) / (df[qt] + 0.5) + 1) for qt in query_tokens}
|
||||||
|
avgdl = sum(len(d["tokens"]) for d in corpus) / max(n, 1)
|
||||||
|
scored = []
|
||||||
|
for d in corpus:
|
||||||
|
dl = len(d["tokens"])
|
||||||
|
score = 0.0
|
||||||
|
for qt in set(query_tokens):
|
||||||
|
if qt in d["tokens"]:
|
||||||
|
f = d["tokens"].count(qt)
|
||||||
|
score += idf[qt] * (f * (k1 + 1)) / (f + k1 * (1 - b + b * (dl / max(avgdl, 1))))
|
||||||
|
if score > 0:
|
||||||
|
scored.append({**d, "bm25": score})
|
||||||
|
scored.sort(key=lambda x: x["bm25"], reverse=True)
|
||||||
|
return scored
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Level 1:知识库目录树
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def build_kb_tree(db, user_id: Optional[str] = None, kb_id: Optional[str] = None) -> List[Dict]:
|
||||||
|
"""构造 知识库 -> 目录(可嵌套) -> 文档 的结构树。"""
|
||||||
|
from app.models import KnowledgeBase, KnowledgeDirectory, KnowledgeDocument
|
||||||
|
|
||||||
|
q = db.query(KnowledgeBase)
|
||||||
|
if user_id:
|
||||||
|
q = q.filter(KnowledgeBase.user_id == user_id)
|
||||||
|
if kb_id:
|
||||||
|
q = q.filter(KnowledgeBase.id == kb_id)
|
||||||
|
bases = q.all()
|
||||||
|
|
||||||
|
tree = []
|
||||||
|
for kb in bases:
|
||||||
|
dirs = db.query(KnowledgeDirectory).filter(
|
||||||
|
KnowledgeDirectory.knowledge_base_id == kb.id
|
||||||
|
).all()
|
||||||
|
docs = db.query(KnowledgeDocument).filter(
|
||||||
|
KnowledgeDocument.knowledge_base_id == kb.id,
|
||||||
|
KnowledgeDocument.status == "completed",
|
||||||
|
).all()
|
||||||
|
dir_by_id = {d.id: d for d in dirs}
|
||||||
|
node_by_id = {}
|
||||||
|
|
||||||
|
def make_dir_node(d):
|
||||||
|
return {
|
||||||
|
"id": d.id,
|
||||||
|
"name": d.name,
|
||||||
|
"type": "directory",
|
||||||
|
"children": [],
|
||||||
|
"documents": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
for d in dirs:
|
||||||
|
node_by_id[d.id] = make_dir_node(d)
|
||||||
|
root_dirs = []
|
||||||
|
for d in dirs:
|
||||||
|
node = node_by_id[d.id]
|
||||||
|
parent = d.parent_id
|
||||||
|
if parent and parent in node_by_id:
|
||||||
|
node_by_id[parent]["children"].append(node)
|
||||||
|
else:
|
||||||
|
root_dirs.append(node)
|
||||||
|
for doc in docs:
|
||||||
|
item = {
|
||||||
|
"id": doc.id,
|
||||||
|
"name": doc.filename,
|
||||||
|
"type": "document",
|
||||||
|
"file_type": doc.file_type,
|
||||||
|
"chunk_count": doc.chunk_count or 0,
|
||||||
|
}
|
||||||
|
if doc.directory_id and doc.directory_id in node_by_id:
|
||||||
|
node_by_id[doc.directory_id]["documents"].append(item)
|
||||||
|
else:
|
||||||
|
root_dirs.append(item) # 顶层文档
|
||||||
|
tree.append({
|
||||||
|
"id": kb.id,
|
||||||
|
"name": kb.name,
|
||||||
|
"type": "knowledge_base",
|
||||||
|
"directories": root_dirs,
|
||||||
|
"document_count": len(docs),
|
||||||
|
})
|
||||||
|
return tree
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 面包屑:KB名 -> 目录路径 -> 文件名 -> 段标题/序号
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def _breadcrumb(db, kb_id: str, directory_id: Optional[str], doc_id: str, chunk_index: int, title: str) -> Dict:
|
||||||
|
from app.models import KnowledgeBase, KnowledgeDirectory, KnowledgeDocument
|
||||||
|
kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first()
|
||||||
|
kb_name = kb.name if kb else kb_id
|
||||||
|
path = []
|
||||||
|
cur = directory_id
|
||||||
|
guard = 0
|
||||||
|
while cur and guard < 20:
|
||||||
|
d = db.query(KnowledgeDirectory).filter(KnowledgeDirectory.id == cur).first()
|
||||||
|
if not d:
|
||||||
|
break
|
||||||
|
path.insert(0, d.name)
|
||||||
|
cur = d.parent_id
|
||||||
|
guard += 1
|
||||||
|
doc = db.query(KnowledgeDocument).filter(KnowledgeDocument.id == doc_id).first()
|
||||||
|
doc_name = doc.filename if doc else doc_id
|
||||||
|
return {
|
||||||
|
"knowledge_base": kb_name,
|
||||||
|
"directory_path": "/".join(path) if path else "/",
|
||||||
|
"document": doc_name,
|
||||||
|
"chunk_index": chunk_index,
|
||||||
|
"chunk_title": title,
|
||||||
|
"label": f"{kb_name}/{('/'.join(path) if path else '')}/{doc_name}" + (f"#段{chunk_index}" + (f"·{title}" if title else "")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Level 2:结构化分片段检索
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
async def tree_search(
|
||||||
|
db,
|
||||||
|
query: str,
|
||||||
|
user_id: Optional[str] = None,
|
||||||
|
kb_ids: Optional[List[str]] = None,
|
||||||
|
scope_base: Optional[str] = None,
|
||||||
|
scope_dir: Optional[str] = None,
|
||||||
|
scope_doc: Optional[str] = None,
|
||||||
|
top_k: int = 5,
|
||||||
|
mode: str = "hybrid",
|
||||||
|
max_tokens: int = 1200,
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
两级检索的 Level 2:在作用域内做分片段精准检索。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query : 查询文本
|
||||||
|
kb_ids : 限定知识库范围(多个)
|
||||||
|
scope_base : 限定单一知识库
|
||||||
|
scope_dir : 限定单一目录(含子目录)
|
||||||
|
scope_doc : 限定单一文档
|
||||||
|
top_k : 返回命中数
|
||||||
|
mode : "bm25" | "vector" | "hybrid"(默认)
|
||||||
|
Returns:
|
||||||
|
[{
|
||||||
|
"content": <分段文本>,
|
||||||
|
"score": 0-1,
|
||||||
|
"source": "tree_chunk",
|
||||||
|
"metadata": { "breadcrumb": {...}, "snippet": ..., "chunk_index": int, "title": str }
|
||||||
|
}, ...]
|
||||||
|
"""
|
||||||
|
from app.models import KnowledgeBase, KnowledgeDirectory, KnowledgeDocument, KnowledgeDocumentChunk
|
||||||
|
|
||||||
|
if not query or not query.strip():
|
||||||
|
return []
|
||||||
|
|
||||||
|
# 1) 解析作用域 -> 目标文档 id 集合
|
||||||
|
target_kb_ids = list(kb_ids or [])
|
||||||
|
if scope_base and scope_base not in target_kb_ids:
|
||||||
|
target_kb_ids.append(scope_base)
|
||||||
|
|
||||||
|
doc_scope = set()
|
||||||
|
if scope_doc:
|
||||||
|
doc_scope.add(scope_doc)
|
||||||
|
if scope_dir:
|
||||||
|
# 收集该目录及其所有子目录下的文档
|
||||||
|
dirs = db.query(KnowledgeDirectory).all()
|
||||||
|
dir_by_id = {d.id: d for d in dirs}
|
||||||
|
sub = set()
|
||||||
|
|
||||||
|
def collect(did):
|
||||||
|
sub.add(did)
|
||||||
|
for d in dirs:
|
||||||
|
if d.parent_id == did:
|
||||||
|
collect(d.id)
|
||||||
|
collect(scope_dir)
|
||||||
|
docs = db.query(KnowledgeDocument).filter(KnowledgeDocument.directory_id.in_(sub)).all()
|
||||||
|
for d in docs:
|
||||||
|
doc_scope.add(d.id)
|
||||||
|
if not doc_scope and not target_kb_ids and not user_id:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# 2) 取分段语料
|
||||||
|
q = db.query(KnowledgeDocumentChunk)
|
||||||
|
if doc_scope:
|
||||||
|
q = q.filter(KnowledgeDocumentChunk.document_id.in_(doc_scope))
|
||||||
|
else:
|
||||||
|
if target_kb_ids:
|
||||||
|
q = q.filter(KnowledgeDocumentChunk.knowledge_base_id.in_(target_kb_ids))
|
||||||
|
elif user_id:
|
||||||
|
kb_ids_u = [r[0] for r in db.query(KnowledgeBase.id).filter(KnowledgeBase.user_id == user_id).all()]
|
||||||
|
if not kb_ids_u:
|
||||||
|
return []
|
||||||
|
q = q.filter(KnowledgeDocumentChunk.knowledge_base_id.in_(kb_ids_u))
|
||||||
|
chunks = q.order_by(KnowledgeDocumentChunk.chunk_index).limit(500).all()
|
||||||
|
if not chunks:
|
||||||
|
return []
|
||||||
|
|
||||||
|
query_tokens = _tokenize(query)
|
||||||
|
corpus = []
|
||||||
|
for c in chunks:
|
||||||
|
text = _chunk_display_text(c)
|
||||||
|
if len(text.strip()) < 8:
|
||||||
|
continue
|
||||||
|
corpus.append({
|
||||||
|
"id": c.id,
|
||||||
|
"tokens": _tokenize(text),
|
||||||
|
"chunk": c,
|
||||||
|
"text": text,
|
||||||
|
})
|
||||||
|
|
||||||
|
# 3) BM25 排序
|
||||||
|
bm25_hits = _bm25_rank(query_tokens, corpus) if mode in ("bm25", "hybrid") else []
|
||||||
|
|
||||||
|
# 4) 可选向量 boost
|
||||||
|
vector_boost = {}
|
||||||
|
if mode in ("vector", "hybrid"):
|
||||||
|
try:
|
||||||
|
from app.pgvector_search import vector_search
|
||||||
|
for kid in (target_kb_ids or [None]):
|
||||||
|
try:
|
||||||
|
vres = await vector_search(query, kid, top_k=top_k * 3)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[tree_search] vector_search failed: {e}")
|
||||||
|
continue
|
||||||
|
for vr in vres:
|
||||||
|
vcontent = (vr.get("content") or "").strip()
|
||||||
|
if vcontent:
|
||||||
|
vector_boost[vcontent] = max(vector_boost.get(vcontent, 0.0), float(vr.get("score", 0.0)))
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"[tree_search] vector unavailable: {e}")
|
||||||
|
|
||||||
|
# 5) 融合
|
||||||
|
merged_map = {}
|
||||||
|
for rank, h in enumerate(bm25_hits):
|
||||||
|
key = h["text"][:300]
|
||||||
|
boost = 0.0
|
||||||
|
for vc, vs in vector_boost.items():
|
||||||
|
if vc and vc in h["text"]:
|
||||||
|
boost = max(boost, vs)
|
||||||
|
merged_map[key] = {
|
||||||
|
"chunk": h["chunk"],
|
||||||
|
"bm25": h["bm25"],
|
||||||
|
"vec": boost,
|
||||||
|
"rank": rank + 1,
|
||||||
|
}
|
||||||
|
# 纯向量模式:把仅在向量里命中的也纳入
|
||||||
|
if mode == "vector":
|
||||||
|
for vc, vs in vector_boost.items():
|
||||||
|
if vc[:300] not in merged_map:
|
||||||
|
# 找到对应 chunk
|
||||||
|
for c in corpus:
|
||||||
|
if vc in c["text"]:
|
||||||
|
merged_map[vc[:300]] = {"chunk": c["chunk"], "bm25": 0.0, "vec": vs, "rank": 999}
|
||||||
|
break
|
||||||
|
|
||||||
|
if not merged_map:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# 综合分数(BM25 主导,向量做 0~1 归一 boost)
|
||||||
|
max_bm25 = max((m["bm25"] for m in merged_map.values()), default=1.0) or 1.0
|
||||||
|
items = []
|
||||||
|
for key, m in merged_map.items():
|
||||||
|
c = m["chunk"]
|
||||||
|
norm_bm25 = (m["bm25"] / max_bm25) if max_bm25 > 0 else 0.0
|
||||||
|
norm_vec = min(m["vec"], 1.0)
|
||||||
|
combined = norm_bm25 * 0.7 + norm_vec * 0.3
|
||||||
|
title = _chunk_title(c)
|
||||||
|
breadcrumb = _breadcrumb(db, c.knowledge_base_id, c.document_id and _doc_dir(db, c.document_id), c.document_id, c.chunk_index, title)
|
||||||
|
snippet = _make_snippet(c.content or "", query)
|
||||||
|
items.append({
|
||||||
|
"content": c.content or "",
|
||||||
|
"score": round(combined, 4),
|
||||||
|
"source": "tree_chunk",
|
||||||
|
"metadata": {
|
||||||
|
"breadcrumb": breadcrumb,
|
||||||
|
"snippet": snippet,
|
||||||
|
"chunk_index": c.chunk_index,
|
||||||
|
"title": title,
|
||||||
|
"document_id": c.document_id,
|
||||||
|
"knowledge_base_id": c.knowledge_base_id,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
items.sort(key=lambda x: x["score"], reverse=True)
|
||||||
|
return items[:top_k]
|
||||||
|
|
||||||
|
|
||||||
|
def _doc_dir(db, doc_id: str):
|
||||||
|
from app.models import KnowledgeDocument
|
||||||
|
d = db.query(KnowledgeDocument).filter(KnowledgeDocument.id == doc_id).first()
|
||||||
|
return d.directory_id if d else None
|
||||||
|
|
||||||
|
|
||||||
|
def _make_snippet(content: str, query: str, win: int = 80) -> str:
|
||||||
|
if not content:
|
||||||
|
return ""
|
||||||
|
q = query.strip()
|
||||||
|
idx = content.find(q)
|
||||||
|
if idx < 0:
|
||||||
|
# 尝试首个 token
|
||||||
|
toks = _tokenize(q)
|
||||||
|
for t in toks:
|
||||||
|
idx = content.find(t)
|
||||||
|
if idx >= 0:
|
||||||
|
break
|
||||||
|
if idx < 0:
|
||||||
|
return content[:win * 2]
|
||||||
|
start = max(0, idx - win)
|
||||||
|
end = min(len(content), idx + len(q) + win)
|
||||||
|
snip = content[start:end]
|
||||||
|
if start > 0:
|
||||||
|
snip = "..." + snip
|
||||||
|
if end < len(content):
|
||||||
|
snip = snip + "..."
|
||||||
|
return snip
|
||||||
Reference in new issue
Block a user