import os, re, difflib
from collections import Counter

base = "/app/working/workspaces/default/novels/r2-xi-tong-nei-juan"
files = sorted([f for f in os.listdir(base) if f.endswith(".md") and f != "README.md"])

chapters = {}
for f in files:
    with open(os.path.join(base, f)) as fh:
        chapters[f] = fh.read()

# 1. 段落级重复
print("=== 段落级重复检测 ===")
segments = []
for f in files:
    for para in chapters[f].split("\n\n"):
        s = para.strip()
        lines = s.split("\n")
        clean = [l for l in lines if not l.startswith("# ") and len(l.strip()) > 10]
        if clean:
            segments.append(" ".join(clean)[:200])

ct = Counter(segments)
dupes = [(k, v) for k, v in ct.items() if v >= 2]
if dupes:
    print("发现重复段落:")
    for k, v in dupes[:10]:
        print(f"  x{v} : {k[:80]}...")
else:
    print("无段落级重复")

# 2. 句式级重复
print("\n=== 句式级重复检测 ===")
sentences = []
for f in files:
    for m in re.finditer(r'[^\u3002\uff01\uff1f]*[\u3002\uff01\uff1f]', chapters[f]):
        s = m.group().strip()
        if 10 < len(s) < 80:
            sentences.append(s)
ct2 = Counter(sentences)
hf = [(k, v) for k, v in ct2.items() if v >= 2]
hf.sort(key=lambda x: -x[1])
if hf:
    print(f"发现{len(hf)}个重复句式")
    for k, v in hf[:15]:
        print(f"  x{v} : {k[:60]}")
else:
    print("无句式级重复")

# 3. 卷爷对话重复
print("\n=== 卷爷说话统计 ===")
all_juanye = []
for f in files:
    msgs = re.findall(r'[「『][^」』]*卷爷[^」』]*[」』]', chapters[f])
    all_juanye.extend(msgs)
print(f"卷爷发言总数: {len(all_juanye)}")
ct3 = Counter(all_juanye)
for k, v in ct3.most_common(5):
    print(f"  x{v} : {k[:60]}")

# 4. 相同句式开头 —— 番茄很容易判定为模板化
print("\n=== 章节开头模式检测 ===")
for f in files:
    lines = chapters[f].strip().split("\n")
    # 找第一个非标题行
    for l in lines:
        s = l.strip()
        if s and not s.startswith("# ") and len(s) > 5:
            print(f"  {f[:15]}: {s[:60]}")
            break

print("\n=== 系统提示框重复 ===")
all_boxes = []
for f in files:
    boxes = re.findall(r'「[^」]*\*\*[^」]*\*\*[^」]*」', chapters[f])
    all_boxes.extend(boxes)
ct4 = Counter(all_boxes)
for k, v in ct4.most_common(5):
    if v >= 2:
        print(f"  x{v} : {k[:60]}")
if not any(v >= 2 for _, v in ct4.most_common(5)):
    print("无系统提示框重复")