v0.12: stabilize search routing and profile-driven phase4 pipeline
This commit is contained in:
kai
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4 replacement pipeline orchestrator.
|
||||
|
||||
Default flow:
|
||||
1) translate.py
|
||||
2) optional glossary verification (low-confidence/full/off)
|
||||
3) apply_glossary.py
|
||||
4) polish.py
|
||||
5) build_report.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.markdown_chunker import split_by_headers
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p.resolve()
|
||||
cand = REPO_ROOT / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand.resolve()
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def run_step(cmd: list[str], *, dry_run: bool) -> int:
|
||||
print("$ " + " ".join(cmd))
|
||||
if dry_run:
|
||||
return 0
|
||||
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
|
||||
|
||||
|
||||
def infer_workers(source_file: Path, fallback: int, cap: int = 8) -> int:
|
||||
if not source_file.exists():
|
||||
return fallback
|
||||
text = source_file.read_text(encoding="utf-8")
|
||||
blocks = split_by_headers(text, max_level=2)
|
||||
if not blocks:
|
||||
return fallback
|
||||
cpu_cap = max(2, min(cap, (os.cpu_count() or 4)))
|
||||
suggested = max(2, min(cpu_cap, (len(blocks) + 5) // 6))
|
||||
return max(1, suggested if fallback <= 0 else min(max(fallback, 1), cpu_cap))
|
||||
|
||||
|
||||
def low_confidence_terms(glossary_path: Path) -> list[str]:
|
||||
if not glossary_path.exists():
|
||||
return []
|
||||
try:
|
||||
glossary = json.loads(glossary_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return []
|
||||
out: list[str] = []
|
||||
for term, entry in glossary.items():
|
||||
if not isinstance(entry, dict):
|
||||
out.append(term)
|
||||
continue
|
||||
conf = str(entry.get("confidence", "")).lower()
|
||||
verified = bool(entry.get("verified_at"))
|
||||
if conf != "high" or not verified:
|
||||
out.append(term)
|
||||
return sorted(set(out))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 4 replacement pipeline")
|
||||
parser.add_argument("project", help="Project slug or full path")
|
||||
parser.add_argument("--translate-model", default="anthropic/claude-sonnet-4.6")
|
||||
parser.add_argument("--glossary-model", default="anthropic/claude-haiku-4.5")
|
||||
parser.add_argument("--polish-model", default="anthropic/claude-sonnet-4.6")
|
||||
parser.add_argument("--translate-workers", type=int, default=0, help="0 means auto")
|
||||
parser.add_argument("--glossary-workers", type=int, default=4)
|
||||
parser.add_argument("--polish-workers", type=int, default=0, help="0 means auto")
|
||||
parser.add_argument(
|
||||
"--glossary-mode",
|
||||
choices=["off", "low-confidence", "full"],
|
||||
default="low-confidence",
|
||||
help="off: skip, low-confidence: verify only low-confidence terms, full: verify all",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
project_root = resolve_project(args.project)
|
||||
phase4 = project_root / "phase4"
|
||||
src_en = phase4 / "final_en.md"
|
||||
if not src_en.exists():
|
||||
raise SystemExit(f"missing source: {src_en}")
|
||||
|
||||
tw = infer_workers(src_en, args.translate_workers)
|
||||
zh = phase4 / "final_zh.md"
|
||||
pw = infer_workers(zh if zh.exists() else src_en, args.polish_workers)
|
||||
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Translate workers: {tw} | Polish workers: {pw}")
|
||||
print(f"Glossary mode: {args.glossary_mode}")
|
||||
print()
|
||||
|
||||
t0 = time.time()
|
||||
steps: list[list[str]] = [
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "translate.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(tw),
|
||||
"--model",
|
||||
args.translate_model,
|
||||
]
|
||||
]
|
||||
|
||||
if args.glossary_mode != "off":
|
||||
gcmd = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_glossary.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(args.glossary_workers),
|
||||
"--model",
|
||||
args.glossary_model,
|
||||
]
|
||||
if args.glossary_mode == "low-confidence":
|
||||
terms = low_confidence_terms(phase4 / "glossary.json")
|
||||
if terms:
|
||||
if len(terms) > 80:
|
||||
print(f"[info] low-confidence terms={len(terms)} is large; fallback to full glossary verify")
|
||||
else:
|
||||
gcmd += ["--only", ",".join(terms)]
|
||||
else:
|
||||
print("[info] no low-confidence glossary terms found; skipping glossary step")
|
||||
gcmd = []
|
||||
if gcmd:
|
||||
steps.append(gcmd)
|
||||
|
||||
steps.extend(
|
||||
[
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "apply_glossary.py"),
|
||||
str(project_root),
|
||||
"--input",
|
||||
"phase4/final_zh.md",
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "polish.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(pw),
|
||||
"--model",
|
||||
args.polish_model,
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_report.py"),
|
||||
str(project_root),
|
||||
],
|
||||
]
|
||||
)
|
||||
|
||||
for cmd in steps:
|
||||
rc = run_step(cmd, dry_run=args.dry_run)
|
||||
if rc != 0:
|
||||
return rc
|
||||
|
||||
print(f"\nPhase 4 pipeline done in {time.time() - t0:.1f}s")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user