83 lines
2.4 KiB
Python
83 lines
2.4 KiB
Python
import os, re, difflib
|
|
from collections import Counter
|
|
|
|
base = "/app/working/workspaces/default/novels/r2-xi-tong-nei-juan"
|
|
files = sorted([f for f in os.listdir(base) if f.endswith(".md") and f != "README.md"])
|
|
|
|
chapters = {}
|
|
for f in files:
|
|
with open(os.path.join(base, f)) as fh:
|
|
chapters[f] = fh.read()
|
|
|
|
# 1. 段落级重复
|
|
print("=== 段落级重复检测 ===")
|
|
segments = []
|
|
for f in files:
|
|
for para in chapters[f].split("\n\n"):
|
|
s = para.strip()
|
|
lines = s.split("\n")
|
|
clean = [l for l in lines if not l.startswith("# ") and len(l.strip()) > 10]
|
|
if clean:
|
|
segments.append(" ".join(clean)[:200])
|
|
|
|
ct = Counter(segments)
|
|
dupes = [(k, v) for k, v in ct.items() if v >= 2]
|
|
if dupes:
|
|
print("发现重复段落:")
|
|
for k, v in dupes[:10]:
|
|
print(f" x{v} : {k[:80]}...")
|
|
else:
|
|
print("无段落级重复")
|
|
|
|
# 2. 句式级重复
|
|
print("\n=== 句式级重复检测 ===")
|
|
sentences = []
|
|
for f in files:
|
|
for m in re.finditer(r'[^\u3002\uff01\uff1f]*[\u3002\uff01\uff1f]', chapters[f]):
|
|
s = m.group().strip()
|
|
if 10 < len(s) < 80:
|
|
sentences.append(s)
|
|
ct2 = Counter(sentences)
|
|
hf = [(k, v) for k, v in ct2.items() if v >= 2]
|
|
hf.sort(key=lambda x: -x[1])
|
|
if hf:
|
|
print(f"发现{len(hf)}个重复句式")
|
|
for k, v in hf[:15]:
|
|
print(f" x{v} : {k[:60]}")
|
|
else:
|
|
print("无句式级重复")
|
|
|
|
# 3. 卷爷对话重复
|
|
print("\n=== 卷爷说话统计 ===")
|
|
all_juanye = []
|
|
for f in files:
|
|
msgs = re.findall(r'[「『][^」』]*卷爷[^」』]*[」』]', chapters[f])
|
|
all_juanye.extend(msgs)
|
|
print(f"卷爷发言总数: {len(all_juanye)}")
|
|
ct3 = Counter(all_juanye)
|
|
for k, v in ct3.most_common(5):
|
|
print(f" x{v} : {k[:60]}")
|
|
|
|
# 4. 相同句式开头 —— 番茄很容易判定为模板化
|
|
print("\n=== 章节开头模式检测 ===")
|
|
for f in files:
|
|
lines = chapters[f].strip().split("\n")
|
|
# 找第一个非标题行
|
|
for l in lines:
|
|
s = l.strip()
|
|
if s and not s.startswith("# ") and len(s) > 5:
|
|
print(f" {f[:15]}: {s[:60]}")
|
|
break
|
|
|
|
print("\n=== 系统提示框重复 ===")
|
|
all_boxes = []
|
|
for f in files:
|
|
boxes = re.findall(r'「[^」]*\*\*[^」]*\*\*[^」]*」', chapters[f])
|
|
all_boxes.extend(boxes)
|
|
ct4 = Counter(all_boxes)
|
|
for k, v in ct4.most_common(5):
|
|
if v >= 2:
|
|
print(f" x{v} : {k[:60]}")
|
|
if not any(v >= 2 for _, v in ct4.most_common(5)):
|
|
print("无系统提示框重复")
|