deep_research/scripts/phase4_pipeline.py

#!/usr/bin/env python3
"""Phase 4 replacement pipeline orchestrator.

Default flow:
1) translate.py
2) optional glossary verification (low-confidence/full/off)
3) apply_glossary.py
4) polish.py
5) build_report.py
"""

from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path


REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(REPO_ROOT))

from scripts.lib.markdown_chunker import split_by_headers


def resolve_project(arg: str) -> Path:
    p = Path(arg)
    if p.is_dir():
        return p.resolve()
    cand = REPO_ROOT / "projects" / arg
    if cand.is_dir():
        return cand.resolve()
    raise SystemExit(f"project not found: {arg}")


def run_step(cmd: list[str], *, dry_run: bool) -> int:
    print("$ " + " ".join(cmd))
    if dry_run:
        return 0
    return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode


def infer_workers(source_file: Path, fallback: int, cap: int = 8) -> int:
    if not source_file.exists():
        return fallback
    text = source_file.read_text(encoding="utf-8")
    blocks = split_by_headers(text, max_level=2)
    if not blocks:
        return fallback
    cpu_cap = max(2, min(cap, (os.cpu_count() or 4)))
    suggested = max(2, min(cpu_cap, (len(blocks) + 5) // 6))
    return max(1, suggested if fallback <= 0 else min(max(fallback, 1), cpu_cap))


def low_confidence_terms(glossary_path: Path) -> list[str]:
    if not glossary_path.exists():
        return []
    try:
        glossary = json.loads(glossary_path.read_text(encoding="utf-8"))
    except Exception:
        return []
    out: list[str] = []
    for term, entry in glossary.items():
        if not isinstance(entry, dict):
            out.append(term)
            continue
        conf = str(entry.get("confidence", "")).lower()
        verified = bool(entry.get("verified_at"))
        if conf != "high" or not verified:
            out.append(term)
    return sorted(set(out))


def main() -> int:
    parser = argparse.ArgumentParser(description="Phase 4 replacement pipeline")
    parser.add_argument("project", help="Project slug or full path")
    parser.add_argument("--translate-model", default="anthropic/claude-sonnet-4.6")
    parser.add_argument("--glossary-model", default="anthropic/claude-haiku-4.5")
    parser.add_argument("--polish-model", default="anthropic/claude-sonnet-4.6")
    parser.add_argument("--translate-workers", type=int, default=0, help="0 means auto")
    parser.add_argument("--glossary-workers", type=int, default=4)
    parser.add_argument("--polish-workers", type=int, default=0, help="0 means auto")
    parser.add_argument(
        "--glossary-mode",
        choices=["off", "low-confidence", "full"],
        default="low-confidence",
        help="off: skip, low-confidence: verify only low-confidence terms, full: verify all",
    )
    parser.add_argument("--dry-run", action="store_true")
    args = parser.parse_args()

    project_root = resolve_project(args.project)
    phase4 = project_root / "phase4"
    src_en = phase4 / "final_en.md"
    if not src_en.exists():
        raise SystemExit(f"missing source: {src_en}")

    tw = infer_workers(src_en, args.translate_workers)
    zh = phase4 / "final_zh.md"
    pw = infer_workers(zh if zh.exists() else src_en, args.polish_workers)

    print(f"Project: {project_root.name}")
    print(f"Translate workers: {tw} | Polish workers: {pw}")
    print(f"Glossary mode: {args.glossary_mode}")
    print()

    t0 = time.time()
    steps: list[list[str]] = [
        [
            sys.executable,
            str(REPO_ROOT / "scripts" / "translate.py"),
            str(project_root),
            "--workers",
            str(tw),
            "--model",
            args.translate_model,
        ]
    ]

    if args.glossary_mode != "off":
        gcmd = [
            sys.executable,
            str(REPO_ROOT / "scripts" / "build_glossary.py"),
            str(project_root),
            "--workers",
            str(args.glossary_workers),
            "--model",
            args.glossary_model,
        ]
        if args.glossary_mode == "low-confidence":
            terms = low_confidence_terms(phase4 / "glossary.json")
            if terms:
                if len(terms) > 80:
                    print(f"[info] low-confidence terms={len(terms)} is large; fallback to full glossary verify")
                else:
                    gcmd += ["--only", ",".join(terms)]
            else:
                print("[info] no low-confidence glossary terms found; skipping glossary step")
                gcmd = []
        if gcmd:
            steps.append(gcmd)

    steps.extend(
        [
            [
                sys.executable,
                str(REPO_ROOT / "scripts" / "apply_glossary.py"),
                str(project_root),
                "--input",
                "phase4/final_zh.md",
            ],
            [
                sys.executable,
                str(REPO_ROOT / "scripts" / "polish.py"),
                str(project_root),
                "--workers",
                str(pw),
                "--model",
                args.polish_model,
            ],
            [
                sys.executable,
                str(REPO_ROOT / "scripts" / "build_report.py"),
                str(project_root),
            ],
        ]
    )

    for cmd in steps:
        rc = run_step(cmd, dry_run=args.dry_run)
        if rc != 0:
            return rc

    print(f"\nPhase 4 pipeline done in {time.time() - t0:.1f}s")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())