kai/deep_research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.20 alpha skill-driven python core

Browse Source
This commit is contained in:
kai
2026-05-06 16:26:41 +08:00
parent d1169646b8
commit db626f1d58
87 changed files with 5213 additions and 2865 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/dr.py
+487 -12
View File
@@ -12,6 +12,7 @@ import json
import re
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
@@ -25,25 +26,35 @@ from scripts.lib.model_config import (
parse_model_overrides,
resolve_model_profile,
)
from scripts.runtime.assembly import build_chapter_briefs, run_chapter_assembly_workers
from scripts.runtime.orchestrator import create_phase2_task_cards, write_placeholder_packets
from scripts.runtime.methods import ResearchMethodRegistry
from scripts.runtime.phase1 import create_project, render_framework, write_material_brief
from scripts.runtime.review import build_phase3_critique
from scripts.runtime.roles import resolve_runtime_profile
from scripts.runtime.sources import rebuild_sources_from_packets
from scripts.runtime.skills import SkillRegistry, default_adapter_skill_dirs
from scripts.runtime.tasks import TaskCard
from scripts.runtime.workers import run_packet_workers
PROJECTS_DIR = REPO_ROOT / "projects"
CODEX_COMMANDS_DIR = REPO_ROOT / ".codex" / "commands"
CODEX_COMMAND_TEMPLATES_DIR = REPO_ROOT / "codex_adapter_templates" / "codex" / "commands"
LEGACY_CODEX_COMMANDS_DIR = REPO_ROOT / ".codex" / "commands"
def resolve_project(project: str | None) -> Path:
def resolve_project(project: str | None, *, projects_dir: Path = PROJECTS_DIR) -> Path:
if project:
p = Path(project)
if p.is_dir():
return p.resolve()
cand = PROJECTS_DIR / project
cand = projects_dir / project
if cand.is_dir():
return cand.resolve()
raise SystemExit(f"project not found: {project}")
manifests = sorted(
PROJECTS_DIR.glob("*/manifest.json"),
projects_dir.glob("*/manifest.json"),
key=lambda p: p.stat().st_mtime,
reverse=True,
)
@@ -71,6 +82,43 @@ def file_state(path: Path) -> str:
return "yes" if path.exists() else "no"
def packet_state_counts(project_root: Path) -> dict[str, int]:
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
errors = sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
counts = {
"ready": 0,
"placeholder": 0,
"invalid": 0,
"errors": 0,
"stale_errors": 0,
"total": len(packets),
}
ready_stems: set[str] = set()
for path in packets:
try:
packet = json.loads(path.read_text(encoding="utf-8"))
except Exception:
counts["invalid"] += 1
continue
has_evidence = bool(
packet.get("claims")
or packet.get("evidence_items")
or packet.get("counter_evidence")
or packet.get("source_ids")
)
if has_evidence:
counts["ready"] += 1
ready_stems.add(path.stem)
else:
counts["placeholder"] += 1
for path in errors:
if path.stem in ready_stems:
counts["stale_errors"] += 1
else:
counts["errors"] += 1
return counts
def run_cmd(cmd: list[str], *, dry_run: bool) -> int:
printable = " ".join(cmd)
print(f"$ {printable}")
@@ -79,6 +127,244 @@ def run_cmd(cmd: list[str], *, dry_run: bool) -> int:
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
def cmd_init(args: argparse.Namespace) -> int:
projects_dir = Path(args.projects_dir) if args.projects_dir else PROJECTS_DIR
project_root = create_project(
topic=args.topic,
slug=args.slug,
projects_dir=projects_dir,
method_key=args.method,
report_type=args.report_type,
model_profile=args.profile,
target_words=args.target_words,
input_materials=args.input_material,
)
print(f"Project: {project_root.name}")
print(f"Created: {project_root}")
print("Runtime: python-core-v0.20")
print("Next: run `dr.py frame <project>` to generate phase1/framework.md")
return 0
def cmd_frame(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
if args.dry_run:
manifest = load_manifest(project_root)
method = ResearchMethodRegistry().get(args.method or manifest.get("research_method"))
print(f"Project: {project_root.name}")
print(f"Would write: phase1/framework.md")
print(f"Research method: {method.key}")
print(f"Chapters: {args.chapters}")
return 0
path = render_framework(project_root, method_key=args.method, chapter_count=args.chapters)
print(f"Project: {project_root.name}")
print(f"Wrote: {path.relative_to(project_root)}")
print("Pause: review and approve the framework before Phase 2.")
return 0
def cmd_approve(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
phase1 = manifest.setdefault("phase1", {})
phase1["approved"] = True
phase1["requires_user_interview"] = False
phase1["approved_at"] = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
manifest["updated_at"] = phase1["approved_at"]
(project_root / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(f"Project: {project_root.name}")
print("Phase 1 approved. Phase 2 research is now enabled.")
return 0
def cmd_skills(args: argparse.Namespace) -> int:
registry = SkillRegistry()
if args.skills_cmd == "list":
for name in registry.list_names():
print(name)
return 0
if args.skills_cmd == "validate":
result = registry.validate()
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result["ok"] else 1
if args.skills_cmd == "sync":
targets = [Path(item) for item in args.target] if args.target else default_adapter_skill_dirs()
copied = registry.sync_to(targets, force=True)
print(f"Synced skills: {copied}")
for target in targets:
print(f" {target}")
return 0
raise SystemExit(f"unknown skills command: {args.skills_cmd}")
def cmd_methods(args: argparse.Namespace) -> int:
registry = ResearchMethodRegistry()
if args.methods_cmd == "list":
for name in registry.list_names():
method = registry.get(name)
print(f"{method.key}: {method.name}")
return 0
if args.methods_cmd == "show":
method = registry.get(args.method)
print(json.dumps(method.__dict__, ensure_ascii=False, indent=2))
return 0
raise SystemExit(f"unknown methods command: {args.methods_cmd}")
def cmd_research(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
if not args.force and not (manifest.get("phase1") or {}).get("approved"):
raise SystemExit(
"Phase 1 is not approved. Review phase1/material_brief.md and phase1/framework.md, "
"then run `uv run python scripts/dr.py approve <project>` or pass --force."
)
runtime = resolve_runtime_profile(profile=args.profile)
card_dicts = create_phase2_task_cards(
project_root,
axes=args.axis,
dry_run=args.dry_run,
)
if args.execute_packets and args.dry_run:
raise SystemExit("--execute-packets cannot be combined with --dry-run")
if args.assemble_chapters and args.dry_run:
raise SystemExit("--assemble-chapters cannot be combined with --dry-run")
if args.execute_packets:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
from scripts.runtime.workers import ProjectSearchProvider
load_secrets()
def client_factory(_role):
return ZenMuxClient(log_file=project_root / "phase2" / "logs" / "packets.jsonl")
def search_provider_factory():
return ProjectSearchProvider(strict_specialized=not args.allow_search_fallback)
packet_count = run_packet_workers(
project_root=project_root,
cards=[TaskCard(**item) for item in card_dicts],
runtime=runtime,
client_factory=client_factory,
search_provider_factory=search_provider_factory,
workers=args.workers,
)
elif not (args.build_briefs or args.assemble_chapters):
packet_count = write_placeholder_packets(project_root, card_dicts, dry_run=args.dry_run)
else:
packet_count = len(list((project_root / "phase2" / "packets").glob("*.json")))
brief_count = 0
chapter_count = 0
source_count = None
if args.build_briefs or args.assemble_chapters:
source_count = rebuild_sources_from_packets(project_root)
briefs = build_chapter_briefs(project_root)
brief_count = len(briefs)
if args.assemble_chapters:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
load_secrets()
def chapter_client_factory(_role):
return ZenMuxClient(log_file=project_root / "phase2" / "logs" / "chapters.jsonl")
chapter_count = run_chapter_assembly_workers(
project_root=project_root,
briefs=briefs,
runtime=runtime,
client_factory=chapter_client_factory,
workers=args.workers,
)
print(f"Project: {project_root.name}")
print(f"Runtime: python-core-v0.20")
print(f"Model profile: {runtime.profile}")
print(f"Workers: {args.workers}")
print(f"Task cards: {len(card_dicts)}")
print(f"Packets: {packet_count}")
if source_count is not None:
print(f"Sources rebuilt: {source_count}")
if args.build_briefs or args.assemble_chapters:
print(f"Chapter briefs: {brief_count}")
if args.assemble_chapters:
print(f"Chapter drafts: {chapter_count}")
if args.dry_run:
print("Dry run: no files written")
elif args.assemble_chapters:
print("Wrote: phase2/drafts/chXX.md")
elif args.execute_packets:
print("Wrote: phase2/task_cards.json and validated phase2/packets/*.json")
print("Next: rerun with --build-briefs to aggregate packets into chapter briefs.")
elif args.build_briefs:
print("Wrote: phase2/chapter_briefs/*.json")
print("Next: rerun with --assemble-chapters to write Chinese chapter drafts.")
else:
print("Wrote: phase2/task_cards.json and phase2/packets/*.json")
print("Next: rerun with --execute-packets to fill packets via model workers.")
return 0
def cmd_run(args: argparse.Namespace) -> int:
target = args.project_or_topic
projects_dir = Path(args.projects_dir) if args.projects_dir else PROJECTS_DIR
try:
project_root = resolve_project(target, projects_dir=projects_dir)
except SystemExit:
if args.dry_run:
print(f"New topic detected: {target}")
print("Dry run: would create project and write phase1/framework.md")
return 0
project_root = create_project(
topic=target,
slug=args.slug,
projects_dir=projects_dir,
method_key=args.method,
report_type=args.report_type,
model_profile=args.profile or "medium",
target_words=args.target_words,
input_materials=args.input_material,
)
framework = render_framework(project_root, chapter_count=args.chapters)
print(f"Project: {project_root.name}")
print("Runtime: python-core-v0.20")
print(f"Created: {project_root}")
print(f"Wrote: {framework.relative_to(project_root)}")
print("Pause: review and approve the framework before Phase 2.")
return 0
print(f"Project: {project_root.name}")
print("Runtime: python-core-v0.20")
print("Next command: research")
if args.dry_run:
print("Dry run: would inspect manifest and continue from the next incomplete phase")
return 0
return cmd_research(
argparse.Namespace(
project=str(project_root),
workers=args.workers,
axis=None,
profile=args.profile,
execute_packets=False,
allow_search_fallback=False,
build_briefs=False,
assemble_chapters=False,
force=False,
dry_run=False,
)
)
def cmd_review(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
path = build_phase3_critique(project_root)
print(f"Project: {project_root.name}")
print(f"Wrote: {path.relative_to(project_root)}")
print("Pause: review critique before Phase 4.")
return 0
def cmd_status(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
@@ -86,6 +372,8 @@ def cmd_status(args: argparse.Namespace) -> int:
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
evidence = sorted((project_root / "phase2" / "evidence").glob("ch*-evidence.md"))
task_cards = project_root / "phase2" / "task_cards.json"
packet_counts = packet_state_counts(project_root)
sources = project_root / "phase2" / "sources.jsonl"
final_en = project_root / "phase4" / "final_en.md"
final_zh = project_root / "phase4" / "final_zh.md"
@@ -93,7 +381,8 @@ def cmd_status(args: argparse.Namespace) -> int:
glossary = project_root / "phase4" / "glossary.json"
en_words = count_words(final_en.read_text(encoding="utf-8")) if final_en.exists() else 0
zh_chars = count_chinese_chars(final_zh_polished.read_text(encoding="utf-8")) if final_zh_polished.exists() else 0
zh_source = final_zh_polished if final_zh_polished.exists() else final_zh
zh_chars = count_chinese_chars(zh_source.read_text(encoding="utf-8")) if zh_source.exists() else 0
source_count = 0
if sources.exists():
source_count = sum(1 for line in sources.read_text(encoding="utf-8").splitlines() if line.strip())
@@ -110,6 +399,16 @@ def cmd_status(args: argparse.Namespace) -> int:
print()
print("Artifacts:")
print(f" framework: {file_state(project_root / 'phase1' / 'framework.md')}")
print(f" task_cards.json: {file_state(task_cards)}")
print(
" packets: "
f"ready={packet_counts['ready']} "
f"placeholder={packet_counts['placeholder']} "
f"invalid={packet_counts['invalid']} "
f"errors={packet_counts['errors']} "
f"stale_errors={packet_counts['stale_errors']} "
f"total={packet_counts['total']}"
)
print(f" drafts: {len(drafts)}")
print(f" evidence files: {len(evidence)}")
print(f" sources: {source_count}")
@@ -124,13 +423,13 @@ def cmd_prompt(args: argparse.Namespace) -> int:
name = args.command
if not name.startswith("dr-"):
name = f"dr-{name}"
path = CODEX_COMMANDS_DIR / f"{name}.md"
if not path.exists():
fallback = CODEX_COMMAND_TEMPLATES_DIR / f"{name}.md"
if fallback.exists():
path = fallback
else:
raise SystemExit(f"Codex command template not found: {path}")
candidates = [
CODEX_COMMAND_TEMPLATES_DIR / f"{name}.md",
LEGACY_CODEX_COMMANDS_DIR / f"{name}.md",
]
path = next((candidate for candidate in candidates if candidate.exists()), None)
if path is None:
raise SystemExit(f"Codex command template not found: {candidates[0]}")
text = path.read_text(encoding="utf-8")
if args.argument:
@@ -174,6 +473,60 @@ def cmd_finalize(args: argparse.Namespace) -> int:
raise SystemExit(f"model profile resolution failed: {exc}") from exc
roles = resolved["roles"]
if not args.legacy_translate:
cmd: list[str] = [
sys.executable,
str(REPO_ROOT / "scripts" / "build_report.py"),
str(project_root),
"--input",
args.input,
]
if args.report_engine:
cmd += ["--engine", args.report_engine]
if args.no_docx:
cmd.append("--no-docx")
if args.no_pdf:
cmd.append("--no-pdf")
if args.dry_run:
print("Chinese-native finalize plan:")
print("$ " + " ".join(cmd))
if args.polish:
print(
"$ "
+ " ".join(
[
sys.executable,
str(REPO_ROOT / "scripts" / "polish.py"),
str(project_root),
"--input",
args.input,
"--workers",
str(args.polish_workers),
"--model",
roles.get("polish", "anthropic/claude-sonnet-4.6"),
]
)
)
return 0
if args.polish:
rc = run_cmd(
[
sys.executable,
str(REPO_ROOT / "scripts" / "polish.py"),
str(project_root),
"--input",
args.input,
"--workers",
str(args.polish_workers),
"--model",
roles.get("polish", "anthropic/claude-sonnet-4.6"),
],
dry_run=False,
)
if rc != 0:
return rc
return run_cmd(cmd, dry_run=False)
cmd = [
sys.executable,
str(REPO_ROOT / "scripts" / "phase4_pipeline.py"),
@@ -212,6 +565,49 @@ def cmd_models(args: argparse.Namespace) -> int:
except ModelConfigError as exc:
raise SystemExit(f"model profile resolution failed: {exc}") from exc
if args.probe:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets, normalize_zenmux_model
load_secrets()
results = []
with ZenMuxClient() as client:
for requested_model in sorted(set(resolved["roles"].values())):
api_model = normalize_zenmux_model(requested_model)
try:
content = client.chat_complete(
model=requested_model,
system="Health check.",
user="Reply with OK only.",
temperature=0,
max_tokens=16,
tag=f"models:probe:{requested_model}",
)
results.append({
"requested_model": requested_model,
"api_model": api_model,
"ok": True,
"response": content.strip()[:80],
})
except Exception as exc: # noqa: BLE001 - probe should report every model.
results.append({
"requested_model": requested_model,
"api_model": api_model,
"ok": False,
"error": str(exc)[:500],
})
if args.json:
print(json.dumps({**resolved, "probe": results}, ensure_ascii=False, indent=2))
else:
print(f"Profile: {resolved['profile']}")
print("Model probe:")
for item in results:
status = "ok" if item["ok"] else "fail"
print(f" {status} {item['requested_model']} -> {item['api_model']}")
if not item["ok"]:
print(f" {item['error']}")
return 0 if all(item["ok"] for item in results) else 1
if args.json:
print(json.dumps(resolved, ensure_ascii=False, indent=2))
return 0
@@ -222,6 +618,10 @@ def cmd_models(args: argparse.Namespace) -> int:
print("Roles:")
for role in sorted(resolved["roles"]):
print(f" {role}: {resolved['roles'][role]}")
if resolved.get("task_types"):
print("Task types:")
for task_type in sorted(resolved["task_types"]):
print(f" {task_type}: {resolved['task_types'][task_type]}")
return 0
@@ -245,10 +645,78 @@ def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Deep Research platform-neutral CLI")
sub = parser.add_subparsers(dest="cmd", required=True)
init = sub.add_parser("init", help="Initialize a Python-core research project")
init.add_argument("topic", help="Research topic")
init.add_argument("--slug", help="Project slug")
init.add_argument("--method", help="Research method key")
init.add_argument("--type", dest="report_type", default="research", help="Report type")
init.add_argument("--profile", default="medium", help="Model profile name from configs/models.yaml")
init.add_argument("--target-words", type=int, default=30000)
init.add_argument("--input-material", action="append", default=[], help="Path or note for user-provided material")
init.add_argument("--projects-dir", help="Override projects directory")
init.set_defaults(func=cmd_init)
approve = sub.add_parser("approve", help="Approve Phase 1 gates before Phase 2")
approve.add_argument("project", help="Project slug or path")
approve.set_defaults(func=cmd_approve)
frame = sub.add_parser("frame", help="Generate Phase 1 framework.md")
frame.add_argument("project", help="Project slug or path")
frame.add_argument("--method", help="Override research method key")
frame.add_argument("--chapters", type=int, default=10)
frame.add_argument("--dry-run", action="store_true")
frame.set_defaults(func=cmd_frame)
run = sub.add_parser("run", help="Run the platform-neutral Python-core workflow")
run.add_argument("project_or_topic", help="Project slug/path or new topic")
run.add_argument("--workers", type=int, default=6)
run.add_argument("--profile", help="Model profile name from configs/models.yaml")
run.add_argument("--slug", help="Project slug when project_or_topic is new")
run.add_argument("--method", help="Research method key when project_or_topic is new")
run.add_argument("--type", dest="report_type", default="research", help="Report type for new project")
run.add_argument("--target-words", type=int, default=30000)
run.add_argument("--chapters", type=int, default=10)
run.add_argument("--input-material", action="append", default=[])
run.add_argument("--projects-dir", help="Override projects directory")
run.add_argument("--dry-run", action="store_true")
run.set_defaults(func=cmd_run)
research = sub.add_parser("research", help="Run v0.20 task-card Phase 2")
research.add_argument("project", help="Project slug or path")
research.add_argument("--workers", type=int, default=6)
research.add_argument("--axis", action="append", help="Restrict generated task axes; repeatable")
research.add_argument("--profile", help="Model profile name from configs/models.yaml")
research.add_argument("--execute-packets", action="store_true", help="Call model workers to fill evidence packets")
research.add_argument("--allow-search-fallback", action="store_true", help="Allow generic search fallback for specialized routes")
research.add_argument("--build-briefs", action="store_true", help="Aggregate packets into chapter briefs")
research.add_argument("--assemble-chapters", action="store_true", help="Call model workers to write Chinese chapter drafts")
research.add_argument("--force", action="store_true", help="bypass Phase 1 approval gate")
research.add_argument("--dry-run", action="store_true")
research.set_defaults(func=cmd_research)
skills = sub.add_parser("skills", help="Manage canonical skills")
skill_sub = skills.add_subparsers(dest="skills_cmd", required=True)
skill_sub.add_parser("list", help="List canonical skills").set_defaults(func=cmd_skills)
skill_sub.add_parser("validate", help="Validate canonical skills").set_defaults(func=cmd_skills)
skill_sync = skill_sub.add_parser("sync", help="Sync skills into adapter directories")
skill_sync.add_argument("--target", action="append", help="Target skill directory; repeatable")
skill_sync.set_defaults(func=cmd_skills)
methods = sub.add_parser("methods", help="List and inspect research framework methods")
method_sub = methods.add_subparsers(dest="methods_cmd", required=True)
method_sub.add_parser("list", help="List research methods").set_defaults(func=cmd_methods)
method_show = method_sub.add_parser("show", help="Show one research method")
method_show.add_argument("method")
method_show.set_defaults(func=cmd_methods)
status = sub.add_parser("status", help="Show project status")
status.add_argument("project", nargs="?", help="Project slug or path")
status.set_defaults(func=cmd_status)
review = sub.add_parser("review", help="Run deterministic Phase 3 review")
review.add_argument("project", help="Project slug or path")
review.set_defaults(func=cmd_review)
prompt = sub.add_parser("prompt", help="Print a Codex command prompt template")
prompt.add_argument("command", help="Command name, e.g. dr-frame or frame")
prompt.add_argument("argument", nargs="?", help="Replacement for $ARGUMENTS")
@@ -266,6 +734,12 @@ def build_parser() -> argparse.ArgumentParser:
finalize = sub.add_parser("finalize", help="Run Phase 4 deterministic pipeline")
finalize.add_argument("project", help="Project slug or path")
finalize.add_argument("--input", default="phase4/final_zh.md", help="Chinese Markdown source for default v0.20 finalization")
finalize.add_argument("--legacy-translate", action="store_true", help="Use legacy final_en -> translate -> polish pipeline")
finalize.add_argument("--polish", action="store_true", help="Run optional Chinese polish step before rendering")
finalize.add_argument("--report-engine", choices=["reportlab", "quarto"], default=None)
finalize.add_argument("--no-docx", action="store_true")
finalize.add_argument("--no-pdf", action="store_true")
finalize.add_argument("--translate-workers", type=int, default=0)
finalize.add_argument("--glossary-workers", type=int, default=4)
finalize.add_argument("--polish-workers", type=int, default=0)
@@ -295,6 +769,7 @@ def build_parser() -> argparse.ArgumentParser:
metavar="ROLE=MODEL",
help="Override one role model, repeatable",
)
models.add_argument("--probe", action="store_true", help="Send tiny health checks to resolved role models")
models.add_argument("--json", action="store_true", help="Emit JSON")
models.set_defaults(func=cmd_models)