deep_research/scripts/check_citations.py

#!/usr/bin/env python3
"""引文完整性核查。

检查 final_zh_polished.md（或其它正文）中的 [src_xxx] 引用与 sources.jsonl 是否一致：
- 孤立引用（正文有但 sources.jsonl 无）：需要 dr-analyst 补信源或删这处引用
- 孤岛信源（sources.jsonl 有但正文无）：被 polish 或润色误删了上下文，或 dr-analyst 收集了
  但没用上
- emoji 扫描：正文里不该有 emoji
- 编号格式：检查 src_xxx 是否符合规范

用法：
    uv run python scripts/check_citations.py <project_slug>
    uv run python scripts/check_citations.py <project_slug> --md phase4/final_zh.md
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path


EMOJI_RE = re.compile(
    r"[\U0001F000-\U0001FFFF]"    # Supplementary Plane emoji
    r"|[\u2700-\u27BF]"            # Dingbats (✅ ❌)
    r"|[\u2600-\u26FF]"            # Misc symbols (⭐ ⚠ ☀)
    r"|[\u2B00-\u2BFF]"            # Misc symbols and arrows
)

# 允许的符号（字体支持）
ALLOWED_SYMBOLS = {
    "✓", "×", "◆", "◇", "●", "○", "★", "※",
    "→", "←", "↑", "↓",
}

SRC_ID_RE = re.compile(r"\[(src_[A-Za-z0-9_\-]+(?:\s*,\s*src_[A-Za-z0-9_\-]+)*)\]")


def resolve_project(arg: str) -> Path:
    p = Path(arg)
    if p.is_dir():
        return p
    cand = Path.cwd() / "projects" / arg
    if cand.is_dir():
        return cand
    raise SystemExit(f"project not found: {arg}")


def main() -> int:
    parser = argparse.ArgumentParser(description="引文完整性核查")
    parser.add_argument("project", help="项目 slug 或路径")
    parser.add_argument("--md", default="phase4/final_zh_polished.md")
    parser.add_argument("--sources", default="phase2/sources.jsonl")
    args = parser.parse_args()

    project = resolve_project(args.project)
    md_path = project / args.md
    src_path = project / args.sources

    if not md_path.exists():
        raise SystemExit(f"找不到正文：{md_path}")

    text = md_path.read_text(encoding="utf-8")

    # 1. 收集正文引用（保序去重）
    cited: list[str] = []
    cited_set: set[str] = set()
    for m in SRC_ID_RE.finditer(text):
        for sid in m.group(1).split(","):
            sid = sid.strip()
            if sid and sid not in cited_set:
                cited_set.add(sid)
                cited.append(sid)

    # 2. 收集 sources.jsonl 中的 ID
    sources: dict[str, dict] = {}
    if src_path.exists():
        for line in src_path.read_text(encoding="utf-8").splitlines():
            line = line.strip()
            if not line:
                continue
            try:
                r = json.loads(line)
                if sid := r.get("id"):
                    sources[sid] = r
            except Exception:
                continue

    orphan_cites = [s for s in cited if s not in sources]
    island_sources = [s for s in sources if s not in cited_set]
    matched = [s for s in cited if s in sources]

    print(f"=== 引文统计 ===")
    print(f"  正文引用（独立 ID）：{len(cited_set)}")
    print(f"  sources.jsonl 收录：{len(sources)}")
    print(f"  匹配：{len(matched)}")
    print(f"  孤立引用（正文有 sources 无）：{len(orphan_cites)}")
    print(f"  孤岛信源（sources 有正文无）：{len(island_sources)}")

    if orphan_cites:
        print(f"\n=== 孤立引用（前 20 条）===")
        for s in orphan_cites[:20]:
            print(f"  {s}")
        if len(orphan_cites) > 20:
            print(f"  ...还有 {len(orphan_cites) - 20}")
        print(f"\n  处理建议：")
        print(f"  (a) 如果是 dr-analyst 编造的占位符 → 在正文中删除该引用")
        print(f"  (b) 如果是信源未收录 → 补到 sources.jsonl")

    if island_sources:
        print(f"\n=== 孤岛信源（前 20 条）===")
        for s in island_sources[:20]:
            print(f"  {s}  — {sources[s].get('title', '')[:80]}")
        if len(island_sources) > 20:
            print(f"  ...还有 {len(island_sources) - 20}")
        print(f"\n  处理建议：")
        print(f"  (a) 如果是 polish 阶段误删了使用该信源的段落 → 检查 polish diff")
        print(f"  (b) 如果是收集多余信源 → 可以保留（build_references 会自动忽略）")

    # 3. Emoji 扫描
    emoji_hits = []
    for m in EMOJI_RE.finditer(text):
        c = m.group()
        if c not in ALLOWED_SYMBOLS:
            line = text[:m.start()].count("\n") + 1
            emoji_hits.append((line, c))

    if emoji_hits:
        print(f"\n=== ⚠ 发现 {len(emoji_hits)} 个 emoji（不允许出现在正文）===")
        seen = {}
        for line, c in emoji_hits:
            seen.setdefault(c, []).append(line)
        for c, lines in seen.items():
            print(f"  U+{ord(c):04X} {c!r}  第 {lines[:5]} 行 等 {len(lines)} 处")
        print(f"  建议用 python3 替换：sed -i '' 's/{list(seen.keys())[0]}//g' {md_path}")

    # 返回码：有问题返回非零便于 CI 使用
    if orphan_cites or emoji_hits:
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())