Initial commit

check_dup.py

@@ -0,0 +1,82 @@
+import os, re, difflib
+from collections import Counter
+
+base = "/app/working/workspaces/default/novels/r2-xi-tong-nei-juan"
+files = sorted([f for f in os.listdir(base) if f.endswith(".md") and f != "README.md"])
+
+chapters = {}
+for f in files:
+    with open(os.path.join(base, f)) as fh:
+        chapters[f] = fh.read()
+
+# 1. 段落级重复
+print("=== 段落级重复检测 ===")
+segments = []
+for f in files:
+    for para in chapters[f].split("\n\n"):
+        s = para.strip()
+        lines = s.split("\n")
+        clean = [l for l in lines if not l.startswith("# ") and len(l.strip()) > 10]
+        if clean:
+            segments.append(" ".join(clean)[:200])
+
+ct = Counter(segments)
+dupes = [(k, v) for k, v in ct.items() if v >= 2]
+if dupes:
+    print("发现重复段落:")
+    for k, v in dupes[:10]:
+        print(f"  x{v} : {k[:80]}...")
+else:
+    print("无段落级重复")
+
+# 2. 句式级重复
+print("\n=== 句式级重复检测 ===")
+sentences = []
+for f in files:
+    for m in re.finditer(r'[^\u3002\uff01\uff1f]*[\u3002\uff01\uff1f]', chapters[f]):
+        s = m.group().strip()
+        if 10 < len(s) < 80:
+            sentences.append(s)
+ct2 = Counter(sentences)
+hf = [(k, v) for k, v in ct2.items() if v >= 2]
+hf.sort(key=lambda x: -x[1])
+if hf:
+    print(f"发现{len(hf)}个重复句式")
+    for k, v in hf[:15]:
+        print(f"  x{v} : {k[:60]}")
+else:
+    print("无句式级重复")
+
+# 3. 卷爷对话重复
+print("\n=== 卷爷说话统计 ===")
+all_juanye = []
+for f in files:
+    msgs = re.findall(r'[「『][^」』]*卷爷[^」』]*[」』]', chapters[f])
+    all_juanye.extend(msgs)
+print(f"卷爷发言总数: {len(all_juanye)}")
+ct3 = Counter(all_juanye)
+for k, v in ct3.most_common(5):
+    print(f"  x{v} : {k[:60]}")
+
+# 4. 相同句式开头 —— 番茄很容易判定为模板化
+print("\n=== 章节开头模式检测 ===")
+for f in files:
+    lines = chapters[f].strip().split("\n")
+    # 找第一个非标题行
+    for l in lines:
+        s = l.strip()
+        if s and not s.startswith("# ") and len(s) > 5:
+            print(f"  {f[:15]}: {s[:60]}")
+            break
+
+print("\n=== 系统提示框重复 ===")
+all_boxes = []
+for f in files:
+    boxes = re.findall(r'「[^」]*\*\*[^」]*\*\*[^」]*」', chapters[f])
+    all_boxes.extend(boxes)
+ct4 = Counter(all_boxes)
+for k, v in ct4.most_common(5):
+    if v >= 2:
+        print(f"  x{v} : {k[:60]}")
+if not any(v >= 2 for _, v in ct4.most_common(5)):
+    print("无系统提示框重复")