v0.20 alpha skill-driven python core

This commit is contained in:
kai
2026-05-06 16:26:41 +08:00
parent d1169646b8
commit db626f1d58
87 changed files with 5213 additions and 2865 deletions
+7 -56
View File
@@ -41,6 +41,8 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.lib.zenmux_client import load_secrets # noqa: F401 (为一致性)
from scripts.reporting.fonts import resolve_quarto_fonts
from scripts.reporting.references import build_references_block
REPO_ROOT = Path(__file__).resolve().parent.parent
@@ -172,9 +174,10 @@ def prepare_qmd(
subtitle = manifest.get("report_subtitle", "")
date = manifest.get("date", "")
# 决定字体名称:思源宋体 CN 作正文,思源黑体 CN 作标题
main_font = "Source Han Serif CN"
sans_font = "Source Han Sans CN"
# 决定字体名称:Quarto/xelatex 使用系统字体 family name。
fonts = resolve_quarto_fonts(fonts_dir)
main_font = fonts.main_font
sans_font = fonts.sans_font
# Write LaTeX header file for CJK font setup.
# Using a separate .tex file avoids YAML escape issues with backslashes.
@@ -248,7 +251,7 @@ def prepare_qmd(
)
# Replace REFERENCES placeholder with actual references from sources.jsonl
ref_block = _build_references_block(sources_path, md_text)
ref_block = build_references_block(sources_path, md_text)
md_text = re.sub(
r"\[REFERENCES will be filled.*?\]",
ref_block,
@@ -286,58 +289,6 @@ def prepare_qmd(
output_qmd.write_text(front_matter + md_text, encoding="utf-8")
print(f" .qmd prepared: {output_qmd.name} ({len(wide_ranges)} landscape table(s))")
def _build_references_block(sources_path: Path | None, md_text: str) -> str:
"""从 sources.jsonl 生成参考文献列表,只包含在正文中实际引用的信源。"""
if not sources_path or not sources_path.exists():
return "(参考文献列表:sources.jsonl 未找到)"
# Find cited src_ids
cited = set(re.findall(r"\[src_([a-z0-9_]+)\]", md_text))
if not cited:
return ""
sources: dict[str, dict] = {}
with open(sources_path, encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
sid = obj.get("id", "")
key = sid.replace("src_", "")
if key in cited:
sources[sid] = obj
except json.JSONDecodeError:
pass
if not sources:
return ""
lines = ["## 参考文献\n"]
for sid in sorted(sources.keys()):
s = sources[sid]
authors = ", ".join(s.get("authors", [])) if s.get("authors") else ""
year = s.get("year", "")
title = s.get("title", sid)
venue = s.get("venue", "")
url = s.get("url", "")
entry = f"- **[{sid}]** "
if authors:
entry += f"{authors}. "
if year:
entry += f"({year}). "
entry += f"*{title}*"
if venue:
entry += f". {venue}"
if url:
entry += f". <{url}>"
lines.append(entry)
return "\n".join(lines)
def build_pdf_quarto(
md_path: Path,
manifest: dict,
+185
View File
@@ -0,0 +1,185 @@
#!/usr/bin/env python3
"""Deploy platform adapter templates outside the repository checkout."""
from __future__ import annotations
import argparse
import os
import shutil
import sys
from dataclasses import dataclass, field
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.runtime.skills import SkillRegistry
CODEX_TEMPLATE = REPO_ROOT / "codex_adapter_templates" / "codex"
@dataclass
class DeployResult:
platform: str
target: Path
written: list[Path] = field(default_factory=list)
skipped: list[Path] = field(default_factory=list)
planned: list[Path] = field(default_factory=list)
backups: list[Path] = field(default_factory=list)
def default_codex_home(
*,
env: dict[str, str] | None = None,
user_home: Path | None = None,
) -> Path:
values = os.environ if env is None else env
if values.get("CODEX_HOME"):
return Path(values["CODEX_HOME"]).expanduser()
home = Path.home() if user_home is None else user_home
return home / ".codex"
def copy_tree_contents(
src: Path,
dst: Path,
*,
force: bool,
dry_run: bool = False,
backup_existing: bool = True,
exclude: set[Path] | None = None,
) -> DeployResult:
if not src.exists():
raise FileNotFoundError(f"adapter template source not found: {src}")
result = DeployResult(platform="copy", target=dst)
excluded = exclude or set()
for item in sorted(src.rglob("*")):
rel = item.relative_to(src)
if rel in excluded:
continue
target = dst / rel
if item.is_dir():
if not dry_run:
target.mkdir(parents=True, exist_ok=True)
continue
if target.exists() and not force:
result.skipped.append(target)
continue
result.planned.append(target)
if dry_run:
continue
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists() and force and backup_existing:
backup = target.with_name(f"{target.name}.bak")
shutil.copy2(target, backup)
result.backups.append(backup)
shutil.copy2(item, target)
result.written.append(target)
return result
def _merge_results(platform: str, target: Path, parts: list[DeployResult]) -> DeployResult:
merged = DeployResult(platform=platform, target=target)
for part in parts:
merged.written.extend(part.written)
merged.skipped.extend(part.skipped)
merged.planned.extend(part.planned)
merged.backups.extend(part.backups)
return merged
def copy_registered_skills(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
result = DeployResult(platform="skills", target=dst)
for skill in SkillRegistry().list():
part = copy_tree_contents(skill.path.parent, dst / skill.name, force=force, dry_run=dry_run)
result.written.extend(part.written)
result.skipped.extend(part.skipped)
result.planned.extend(part.planned)
result.backups.extend(part.backups)
return result
def deploy_codex(
*,
target: Path | None = None,
force: bool = False,
skip_skills: bool = False,
dry_run: bool = False,
include_config: bool = False,
repo_root: Path = REPO_ROOT,
) -> DeployResult:
codex_home = (target or default_codex_home()).expanduser()
template = repo_root / "codex_adapter_templates" / "codex"
parts = [
copy_tree_contents(
template,
codex_home,
force=force,
dry_run=dry_run,
exclude=set() if include_config else {Path("config.toml")},
),
]
if not skip_skills:
parts.append(copy_registered_skills(codex_home / "skills", force=force, dry_run=dry_run))
return _merge_results("codex", codex_home, parts)
def print_result(result: DeployResult) -> None:
action = "planned" if result.planned and not result.written else "written"
print(f"{result.platform} adapter deployment")
print(f" target: {result.target}")
print(f" files {action}: {len(result.planned if action == 'planned' else result.written)}")
print(f" files skipped: {len(result.skipped)}")
print(f" backups: {len(result.backups)}")
if result.written:
for path in result.written:
print(f" {path}")
elif result.planned:
for path in result.planned:
print(f" {path}")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Deploy Deep Research surface adapters")
sub = parser.add_subparsers(dest="platform", required=True)
codex = sub.add_parser("codex", help="Deploy Codex adapter to CODEX_HOME or a target directory")
codex.add_argument("--target", type=Path, help="Codex home target; defaults to $CODEX_HOME or ~/.codex")
codex.add_argument("--force", action="store_true", help="overwrite existing files and create .bak backups")
codex.add_argument("--skip-skills", action="store_true", help="do not copy canonical skills into target/skills")
codex.add_argument("--include-config", action="store_true", help="also copy config.toml; off by default to avoid overwriting global Codex config")
codex.add_argument("--dry-run", action="store_true", help="show files that would be written")
return parser
def main() -> int:
args = build_parser().parse_args()
if args.platform == "codex":
result = deploy_codex(
target=args.target,
force=args.force,
skip_skills=args.skip_skills,
dry_run=args.dry_run,
include_config=args.include_config,
)
print_result(result)
print()
print("Run Codex from this repository after deployment:")
if args.include_config:
print(" codex --profile deep-research")
else:
print(" codex")
print("Note: config.toml is not copied by default. Use --include-config only if you want the bundled profile.")
return 0
raise SystemExit(f"unsupported platform: {args.platform}")
if __name__ == "__main__":
raise SystemExit(main())
+37 -18
View File
@@ -17,8 +17,13 @@ except ModuleNotFoundError: # pragma: no cover - Python < 3.11 fallback.
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.deploy_adapters import default_codex_home, deploy_codex
from scripts.runtime.skills import SkillRegistry
CODEX_TEMPLATE = REPO_ROOT / "codex_adapter_templates" / "codex"
CODEX_ROOT = REPO_ROOT / ".codex"
OPENCODE_ROOT = REPO_ROOT / ".opencode"
AGENTS_SKILLS = REPO_ROOT / ".agents" / "skills"
OPENCODE_SKILLS = OPENCODE_ROOT / "skills"
@@ -28,11 +33,14 @@ REQUIRED_PATHS = [
"README.md",
"PLAN.md",
".opencode/opencode.json",
".codex/config.toml",
".codex/agents/dr-pm.toml",
".codex/commands/dr-run.md",
"codex_adapter_templates/codex/config.toml",
"codex_adapter_templates/codex/agents/dr-pm.toml",
"codex_adapter_templates/codex/commands/dr-run.md",
".agents/skills/search-strategy/SKILL.md",
"skills/deep-research/SKILL.md",
"skills/document-ingest/SKILL.md",
"scripts/dr.py",
"scripts/deploy_adapters.py",
"scripts/install_codex_adapter.py",
]
@@ -111,19 +119,26 @@ def check_deployment() -> int:
if not (REPO_ROOT / item).exists():
issues.append(f"missing required path: {item}")
tracked = git_tracked([".codex", ".opencode", ".agents/skills"])
tracked = git_tracked(["codex_adapter_templates", ".opencode", ".agents/skills", "skills"])
legacy_tracked = git_tracked([".codex"])
if legacy_tracked:
warnings.append("legacy .codex files are still tracked; run: git rm -r --cached .codex")
for item in REQUIRED_PATHS:
if item.startswith((".codex/", ".opencode/", ".agents/")) and item not in tracked:
if item.startswith(("codex_adapter_templates/", ".opencode/", ".agents/")) and item not in tracked:
warnings.append(f"not tracked by git: {item}")
skill_files = sorted(AGENTS_SKILLS.glob("*/SKILL.md"))
if len(skill_files) < 10:
issues.append(f"expected at least 10 Codex skills, found {len(skill_files)}")
skills = SkillRegistry().list()
if len(skills) < 10:
issues.append(f"expected at least 10 Codex skills, found {len(skills)}")
check_toml(CODEX_ROOT / "config.toml", issues)
for path in sorted((CODEX_ROOT / "agents").glob("*.toml")):
check_toml(CODEX_TEMPLATE / "config.toml", issues)
for path in sorted((CODEX_TEMPLATE / "agents").glob("*.toml")):
check_toml(path, issues)
codex_home = default_codex_home()
if not (codex_home / "commands" / "dr-run.md").exists():
warnings.append(f"Codex adapter not deployed to {codex_home}; run scripts/deploy_adapters.py codex")
env_values = {key: os.environ.get(key, "") for key in REQUIRED_ENV_KEYS}
env_values.update({k: v for k, v in parse_env(REPO_ROOT / "secrets.env").items() if not env_values.get(k)})
missing_env = [key for key in REQUIRED_ENV_KEYS if not env_values.get(key)]
@@ -132,9 +147,11 @@ def check_deployment() -> int:
print("Deep Research deployment check")
print(f" repo: {REPO_ROOT}")
print(f" codex files tracked: {sum(1 for p in tracked if p.startswith('.codex/'))}")
print(f" codex template files tracked: {sum(1 for p in tracked if p.startswith('codex_adapter_templates/'))}")
print(f" legacy .codex files tracked: {len(legacy_tracked)}")
print(f" codex home: {codex_home}")
print(f" opencode files tracked: {sum(1 for p in tracked if p.startswith('.opencode/'))}")
print(f" codex skills: {len(skill_files)}")
print(f" codex skills: {len(skills)}")
if warnings:
print("\nWarnings:")
@@ -151,9 +168,9 @@ def check_deployment() -> int:
return 0
def repair(force: bool) -> int:
def repair(force: bool, codex_home: Path | None) -> int:
try:
codex_written = copy_tree_contents(CODEX_TEMPLATE, CODEX_ROOT, force=force)
codex_result = deploy_codex(target=codex_home, force=force)
skills_written = copy_tree_contents(OPENCODE_SKILLS, AGENTS_SKILLS, force=force)
except PermissionError as exc:
print(f"repair failed: permission denied: {exc}", file=sys.stderr)
@@ -163,19 +180,21 @@ def repair(force: bool) -> int:
return 1
print("Repair completed.")
print(f" .codex files written: {len(codex_written)}")
print(f" Codex home files written: {len(codex_result.written)}")
print(f" Codex home: {codex_result.target}")
print(f" .agents skills written: {len(skills_written)}")
return check_deployment()
def main() -> int:
parser = argparse.ArgumentParser(description="Check or repair Deep Research deployment files")
parser.add_argument("--repair", action="store_true", help="copy Codex templates and skills into hidden dirs")
parser.add_argument("--repair", action="store_true", help="deploy Codex templates outside the repo and sync skills")
parser.add_argument("--force", action="store_true", help="overwrite existing files during --repair")
parser.add_argument("--codex-home", type=Path, help="Codex home target for --repair; defaults to $CODEX_HOME or ~/.codex")
args = parser.parse_args()
if args.repair:
return repair(force=args.force)
return repair(force=args.force, codex_home=args.codex_home)
return check_deployment()
+487 -12
View File
@@ -12,6 +12,7 @@ import json
import re
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
@@ -25,25 +26,35 @@ from scripts.lib.model_config import (
parse_model_overrides,
resolve_model_profile,
)
from scripts.runtime.assembly import build_chapter_briefs, run_chapter_assembly_workers
from scripts.runtime.orchestrator import create_phase2_task_cards, write_placeholder_packets
from scripts.runtime.methods import ResearchMethodRegistry
from scripts.runtime.phase1 import create_project, render_framework, write_material_brief
from scripts.runtime.review import build_phase3_critique
from scripts.runtime.roles import resolve_runtime_profile
from scripts.runtime.sources import rebuild_sources_from_packets
from scripts.runtime.skills import SkillRegistry, default_adapter_skill_dirs
from scripts.runtime.tasks import TaskCard
from scripts.runtime.workers import run_packet_workers
PROJECTS_DIR = REPO_ROOT / "projects"
CODEX_COMMANDS_DIR = REPO_ROOT / ".codex" / "commands"
CODEX_COMMAND_TEMPLATES_DIR = REPO_ROOT / "codex_adapter_templates" / "codex" / "commands"
LEGACY_CODEX_COMMANDS_DIR = REPO_ROOT / ".codex" / "commands"
def resolve_project(project: str | None) -> Path:
def resolve_project(project: str | None, *, projects_dir: Path = PROJECTS_DIR) -> Path:
if project:
p = Path(project)
if p.is_dir():
return p.resolve()
cand = PROJECTS_DIR / project
cand = projects_dir / project
if cand.is_dir():
return cand.resolve()
raise SystemExit(f"project not found: {project}")
manifests = sorted(
PROJECTS_DIR.glob("*/manifest.json"),
projects_dir.glob("*/manifest.json"),
key=lambda p: p.stat().st_mtime,
reverse=True,
)
@@ -71,6 +82,43 @@ def file_state(path: Path) -> str:
return "yes" if path.exists() else "no"
def packet_state_counts(project_root: Path) -> dict[str, int]:
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
errors = sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
counts = {
"ready": 0,
"placeholder": 0,
"invalid": 0,
"errors": 0,
"stale_errors": 0,
"total": len(packets),
}
ready_stems: set[str] = set()
for path in packets:
try:
packet = json.loads(path.read_text(encoding="utf-8"))
except Exception:
counts["invalid"] += 1
continue
has_evidence = bool(
packet.get("claims")
or packet.get("evidence_items")
or packet.get("counter_evidence")
or packet.get("source_ids")
)
if has_evidence:
counts["ready"] += 1
ready_stems.add(path.stem)
else:
counts["placeholder"] += 1
for path in errors:
if path.stem in ready_stems:
counts["stale_errors"] += 1
else:
counts["errors"] += 1
return counts
def run_cmd(cmd: list[str], *, dry_run: bool) -> int:
printable = " ".join(cmd)
print(f"$ {printable}")
@@ -79,6 +127,244 @@ def run_cmd(cmd: list[str], *, dry_run: bool) -> int:
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
def cmd_init(args: argparse.Namespace) -> int:
projects_dir = Path(args.projects_dir) if args.projects_dir else PROJECTS_DIR
project_root = create_project(
topic=args.topic,
slug=args.slug,
projects_dir=projects_dir,
method_key=args.method,
report_type=args.report_type,
model_profile=args.profile,
target_words=args.target_words,
input_materials=args.input_material,
)
print(f"Project: {project_root.name}")
print(f"Created: {project_root}")
print("Runtime: python-core-v0.20")
print("Next: run `dr.py frame <project>` to generate phase1/framework.md")
return 0
def cmd_frame(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
if args.dry_run:
manifest = load_manifest(project_root)
method = ResearchMethodRegistry().get(args.method or manifest.get("research_method"))
print(f"Project: {project_root.name}")
print(f"Would write: phase1/framework.md")
print(f"Research method: {method.key}")
print(f"Chapters: {args.chapters}")
return 0
path = render_framework(project_root, method_key=args.method, chapter_count=args.chapters)
print(f"Project: {project_root.name}")
print(f"Wrote: {path.relative_to(project_root)}")
print("Pause: review and approve the framework before Phase 2.")
return 0
def cmd_approve(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
phase1 = manifest.setdefault("phase1", {})
phase1["approved"] = True
phase1["requires_user_interview"] = False
phase1["approved_at"] = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
manifest["updated_at"] = phase1["approved_at"]
(project_root / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(f"Project: {project_root.name}")
print("Phase 1 approved. Phase 2 research is now enabled.")
return 0
def cmd_skills(args: argparse.Namespace) -> int:
registry = SkillRegistry()
if args.skills_cmd == "list":
for name in registry.list_names():
print(name)
return 0
if args.skills_cmd == "validate":
result = registry.validate()
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result["ok"] else 1
if args.skills_cmd == "sync":
targets = [Path(item) for item in args.target] if args.target else default_adapter_skill_dirs()
copied = registry.sync_to(targets, force=True)
print(f"Synced skills: {copied}")
for target in targets:
print(f" {target}")
return 0
raise SystemExit(f"unknown skills command: {args.skills_cmd}")
def cmd_methods(args: argparse.Namespace) -> int:
registry = ResearchMethodRegistry()
if args.methods_cmd == "list":
for name in registry.list_names():
method = registry.get(name)
print(f"{method.key}: {method.name}")
return 0
if args.methods_cmd == "show":
method = registry.get(args.method)
print(json.dumps(method.__dict__, ensure_ascii=False, indent=2))
return 0
raise SystemExit(f"unknown methods command: {args.methods_cmd}")
def cmd_research(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
if not args.force and not (manifest.get("phase1") or {}).get("approved"):
raise SystemExit(
"Phase 1 is not approved. Review phase1/material_brief.md and phase1/framework.md, "
"then run `uv run python scripts/dr.py approve <project>` or pass --force."
)
runtime = resolve_runtime_profile(profile=args.profile)
card_dicts = create_phase2_task_cards(
project_root,
axes=args.axis,
dry_run=args.dry_run,
)
if args.execute_packets and args.dry_run:
raise SystemExit("--execute-packets cannot be combined with --dry-run")
if args.assemble_chapters and args.dry_run:
raise SystemExit("--assemble-chapters cannot be combined with --dry-run")
if args.execute_packets:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
from scripts.runtime.workers import ProjectSearchProvider
load_secrets()
def client_factory(_role):
return ZenMuxClient(log_file=project_root / "phase2" / "logs" / "packets.jsonl")
def search_provider_factory():
return ProjectSearchProvider(strict_specialized=not args.allow_search_fallback)
packet_count = run_packet_workers(
project_root=project_root,
cards=[TaskCard(**item) for item in card_dicts],
runtime=runtime,
client_factory=client_factory,
search_provider_factory=search_provider_factory,
workers=args.workers,
)
elif not (args.build_briefs or args.assemble_chapters):
packet_count = write_placeholder_packets(project_root, card_dicts, dry_run=args.dry_run)
else:
packet_count = len(list((project_root / "phase2" / "packets").glob("*.json")))
brief_count = 0
chapter_count = 0
source_count = None
if args.build_briefs or args.assemble_chapters:
source_count = rebuild_sources_from_packets(project_root)
briefs = build_chapter_briefs(project_root)
brief_count = len(briefs)
if args.assemble_chapters:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
load_secrets()
def chapter_client_factory(_role):
return ZenMuxClient(log_file=project_root / "phase2" / "logs" / "chapters.jsonl")
chapter_count = run_chapter_assembly_workers(
project_root=project_root,
briefs=briefs,
runtime=runtime,
client_factory=chapter_client_factory,
workers=args.workers,
)
print(f"Project: {project_root.name}")
print(f"Runtime: python-core-v0.20")
print(f"Model profile: {runtime.profile}")
print(f"Workers: {args.workers}")
print(f"Task cards: {len(card_dicts)}")
print(f"Packets: {packet_count}")
if source_count is not None:
print(f"Sources rebuilt: {source_count}")
if args.build_briefs or args.assemble_chapters:
print(f"Chapter briefs: {brief_count}")
if args.assemble_chapters:
print(f"Chapter drafts: {chapter_count}")
if args.dry_run:
print("Dry run: no files written")
elif args.assemble_chapters:
print("Wrote: phase2/drafts/chXX.md")
elif args.execute_packets:
print("Wrote: phase2/task_cards.json and validated phase2/packets/*.json")
print("Next: rerun with --build-briefs to aggregate packets into chapter briefs.")
elif args.build_briefs:
print("Wrote: phase2/chapter_briefs/*.json")
print("Next: rerun with --assemble-chapters to write Chinese chapter drafts.")
else:
print("Wrote: phase2/task_cards.json and phase2/packets/*.json")
print("Next: rerun with --execute-packets to fill packets via model workers.")
return 0
def cmd_run(args: argparse.Namespace) -> int:
target = args.project_or_topic
projects_dir = Path(args.projects_dir) if args.projects_dir else PROJECTS_DIR
try:
project_root = resolve_project(target, projects_dir=projects_dir)
except SystemExit:
if args.dry_run:
print(f"New topic detected: {target}")
print("Dry run: would create project and write phase1/framework.md")
return 0
project_root = create_project(
topic=target,
slug=args.slug,
projects_dir=projects_dir,
method_key=args.method,
report_type=args.report_type,
model_profile=args.profile or "medium",
target_words=args.target_words,
input_materials=args.input_material,
)
framework = render_framework(project_root, chapter_count=args.chapters)
print(f"Project: {project_root.name}")
print("Runtime: python-core-v0.20")
print(f"Created: {project_root}")
print(f"Wrote: {framework.relative_to(project_root)}")
print("Pause: review and approve the framework before Phase 2.")
return 0
print(f"Project: {project_root.name}")
print("Runtime: python-core-v0.20")
print("Next command: research")
if args.dry_run:
print("Dry run: would inspect manifest and continue from the next incomplete phase")
return 0
return cmd_research(
argparse.Namespace(
project=str(project_root),
workers=args.workers,
axis=None,
profile=args.profile,
execute_packets=False,
allow_search_fallback=False,
build_briefs=False,
assemble_chapters=False,
force=False,
dry_run=False,
)
)
def cmd_review(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
path = build_phase3_critique(project_root)
print(f"Project: {project_root.name}")
print(f"Wrote: {path.relative_to(project_root)}")
print("Pause: review critique before Phase 4.")
return 0
def cmd_status(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
@@ -86,6 +372,8 @@ def cmd_status(args: argparse.Namespace) -> int:
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
evidence = sorted((project_root / "phase2" / "evidence").glob("ch*-evidence.md"))
task_cards = project_root / "phase2" / "task_cards.json"
packet_counts = packet_state_counts(project_root)
sources = project_root / "phase2" / "sources.jsonl"
final_en = project_root / "phase4" / "final_en.md"
final_zh = project_root / "phase4" / "final_zh.md"
@@ -93,7 +381,8 @@ def cmd_status(args: argparse.Namespace) -> int:
glossary = project_root / "phase4" / "glossary.json"
en_words = count_words(final_en.read_text(encoding="utf-8")) if final_en.exists() else 0
zh_chars = count_chinese_chars(final_zh_polished.read_text(encoding="utf-8")) if final_zh_polished.exists() else 0
zh_source = final_zh_polished if final_zh_polished.exists() else final_zh
zh_chars = count_chinese_chars(zh_source.read_text(encoding="utf-8")) if zh_source.exists() else 0
source_count = 0
if sources.exists():
source_count = sum(1 for line in sources.read_text(encoding="utf-8").splitlines() if line.strip())
@@ -110,6 +399,16 @@ def cmd_status(args: argparse.Namespace) -> int:
print()
print("Artifacts:")
print(f" framework: {file_state(project_root / 'phase1' / 'framework.md')}")
print(f" task_cards.json: {file_state(task_cards)}")
print(
" packets: "
f"ready={packet_counts['ready']} "
f"placeholder={packet_counts['placeholder']} "
f"invalid={packet_counts['invalid']} "
f"errors={packet_counts['errors']} "
f"stale_errors={packet_counts['stale_errors']} "
f"total={packet_counts['total']}"
)
print(f" drafts: {len(drafts)}")
print(f" evidence files: {len(evidence)}")
print(f" sources: {source_count}")
@@ -124,13 +423,13 @@ def cmd_prompt(args: argparse.Namespace) -> int:
name = args.command
if not name.startswith("dr-"):
name = f"dr-{name}"
path = CODEX_COMMANDS_DIR / f"{name}.md"
if not path.exists():
fallback = CODEX_COMMAND_TEMPLATES_DIR / f"{name}.md"
if fallback.exists():
path = fallback
else:
raise SystemExit(f"Codex command template not found: {path}")
candidates = [
CODEX_COMMAND_TEMPLATES_DIR / f"{name}.md",
LEGACY_CODEX_COMMANDS_DIR / f"{name}.md",
]
path = next((candidate for candidate in candidates if candidate.exists()), None)
if path is None:
raise SystemExit(f"Codex command template not found: {candidates[0]}")
text = path.read_text(encoding="utf-8")
if args.argument:
@@ -174,6 +473,60 @@ def cmd_finalize(args: argparse.Namespace) -> int:
raise SystemExit(f"model profile resolution failed: {exc}") from exc
roles = resolved["roles"]
if not args.legacy_translate:
cmd: list[str] = [
sys.executable,
str(REPO_ROOT / "scripts" / "build_report.py"),
str(project_root),
"--input",
args.input,
]
if args.report_engine:
cmd += ["--engine", args.report_engine]
if args.no_docx:
cmd.append("--no-docx")
if args.no_pdf:
cmd.append("--no-pdf")
if args.dry_run:
print("Chinese-native finalize plan:")
print("$ " + " ".join(cmd))
if args.polish:
print(
"$ "
+ " ".join(
[
sys.executable,
str(REPO_ROOT / "scripts" / "polish.py"),
str(project_root),
"--input",
args.input,
"--workers",
str(args.polish_workers),
"--model",
roles.get("polish", "anthropic/claude-sonnet-4.6"),
]
)
)
return 0
if args.polish:
rc = run_cmd(
[
sys.executable,
str(REPO_ROOT / "scripts" / "polish.py"),
str(project_root),
"--input",
args.input,
"--workers",
str(args.polish_workers),
"--model",
roles.get("polish", "anthropic/claude-sonnet-4.6"),
],
dry_run=False,
)
if rc != 0:
return rc
return run_cmd(cmd, dry_run=False)
cmd = [
sys.executable,
str(REPO_ROOT / "scripts" / "phase4_pipeline.py"),
@@ -212,6 +565,49 @@ def cmd_models(args: argparse.Namespace) -> int:
except ModelConfigError as exc:
raise SystemExit(f"model profile resolution failed: {exc}") from exc
if args.probe:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets, normalize_zenmux_model
load_secrets()
results = []
with ZenMuxClient() as client:
for requested_model in sorted(set(resolved["roles"].values())):
api_model = normalize_zenmux_model(requested_model)
try:
content = client.chat_complete(
model=requested_model,
system="Health check.",
user="Reply with OK only.",
temperature=0,
max_tokens=16,
tag=f"models:probe:{requested_model}",
)
results.append({
"requested_model": requested_model,
"api_model": api_model,
"ok": True,
"response": content.strip()[:80],
})
except Exception as exc: # noqa: BLE001 - probe should report every model.
results.append({
"requested_model": requested_model,
"api_model": api_model,
"ok": False,
"error": str(exc)[:500],
})
if args.json:
print(json.dumps({**resolved, "probe": results}, ensure_ascii=False, indent=2))
else:
print(f"Profile: {resolved['profile']}")
print("Model probe:")
for item in results:
status = "ok" if item["ok"] else "fail"
print(f" {status} {item['requested_model']} -> {item['api_model']}")
if not item["ok"]:
print(f" {item['error']}")
return 0 if all(item["ok"] for item in results) else 1
if args.json:
print(json.dumps(resolved, ensure_ascii=False, indent=2))
return 0
@@ -222,6 +618,10 @@ def cmd_models(args: argparse.Namespace) -> int:
print("Roles:")
for role in sorted(resolved["roles"]):
print(f" {role}: {resolved['roles'][role]}")
if resolved.get("task_types"):
print("Task types:")
for task_type in sorted(resolved["task_types"]):
print(f" {task_type}: {resolved['task_types'][task_type]}")
return 0
@@ -245,10 +645,78 @@ def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Deep Research platform-neutral CLI")
sub = parser.add_subparsers(dest="cmd", required=True)
init = sub.add_parser("init", help="Initialize a Python-core research project")
init.add_argument("topic", help="Research topic")
init.add_argument("--slug", help="Project slug")
init.add_argument("--method", help="Research method key")
init.add_argument("--type", dest="report_type", default="research", help="Report type")
init.add_argument("--profile", default="medium", help="Model profile name from configs/models.yaml")
init.add_argument("--target-words", type=int, default=30000)
init.add_argument("--input-material", action="append", default=[], help="Path or note for user-provided material")
init.add_argument("--projects-dir", help="Override projects directory")
init.set_defaults(func=cmd_init)
approve = sub.add_parser("approve", help="Approve Phase 1 gates before Phase 2")
approve.add_argument("project", help="Project slug or path")
approve.set_defaults(func=cmd_approve)
frame = sub.add_parser("frame", help="Generate Phase 1 framework.md")
frame.add_argument("project", help="Project slug or path")
frame.add_argument("--method", help="Override research method key")
frame.add_argument("--chapters", type=int, default=10)
frame.add_argument("--dry-run", action="store_true")
frame.set_defaults(func=cmd_frame)
run = sub.add_parser("run", help="Run the platform-neutral Python-core workflow")
run.add_argument("project_or_topic", help="Project slug/path or new topic")
run.add_argument("--workers", type=int, default=6)
run.add_argument("--profile", help="Model profile name from configs/models.yaml")
run.add_argument("--slug", help="Project slug when project_or_topic is new")
run.add_argument("--method", help="Research method key when project_or_topic is new")
run.add_argument("--type", dest="report_type", default="research", help="Report type for new project")
run.add_argument("--target-words", type=int, default=30000)
run.add_argument("--chapters", type=int, default=10)
run.add_argument("--input-material", action="append", default=[])
run.add_argument("--projects-dir", help="Override projects directory")
run.add_argument("--dry-run", action="store_true")
run.set_defaults(func=cmd_run)
research = sub.add_parser("research", help="Run v0.20 task-card Phase 2")
research.add_argument("project", help="Project slug or path")
research.add_argument("--workers", type=int, default=6)
research.add_argument("--axis", action="append", help="Restrict generated task axes; repeatable")
research.add_argument("--profile", help="Model profile name from configs/models.yaml")
research.add_argument("--execute-packets", action="store_true", help="Call model workers to fill evidence packets")
research.add_argument("--allow-search-fallback", action="store_true", help="Allow generic search fallback for specialized routes")
research.add_argument("--build-briefs", action="store_true", help="Aggregate packets into chapter briefs")
research.add_argument("--assemble-chapters", action="store_true", help="Call model workers to write Chinese chapter drafts")
research.add_argument("--force", action="store_true", help="bypass Phase 1 approval gate")
research.add_argument("--dry-run", action="store_true")
research.set_defaults(func=cmd_research)
skills = sub.add_parser("skills", help="Manage canonical skills")
skill_sub = skills.add_subparsers(dest="skills_cmd", required=True)
skill_sub.add_parser("list", help="List canonical skills").set_defaults(func=cmd_skills)
skill_sub.add_parser("validate", help="Validate canonical skills").set_defaults(func=cmd_skills)
skill_sync = skill_sub.add_parser("sync", help="Sync skills into adapter directories")
skill_sync.add_argument("--target", action="append", help="Target skill directory; repeatable")
skill_sync.set_defaults(func=cmd_skills)
methods = sub.add_parser("methods", help="List and inspect research framework methods")
method_sub = methods.add_subparsers(dest="methods_cmd", required=True)
method_sub.add_parser("list", help="List research methods").set_defaults(func=cmd_methods)
method_show = method_sub.add_parser("show", help="Show one research method")
method_show.add_argument("method")
method_show.set_defaults(func=cmd_methods)
status = sub.add_parser("status", help="Show project status")
status.add_argument("project", nargs="?", help="Project slug or path")
status.set_defaults(func=cmd_status)
review = sub.add_parser("review", help="Run deterministic Phase 3 review")
review.add_argument("project", help="Project slug or path")
review.set_defaults(func=cmd_review)
prompt = sub.add_parser("prompt", help="Print a Codex command prompt template")
prompt.add_argument("command", help="Command name, e.g. dr-frame or frame")
prompt.add_argument("argument", nargs="?", help="Replacement for $ARGUMENTS")
@@ -266,6 +734,12 @@ def build_parser() -> argparse.ArgumentParser:
finalize = sub.add_parser("finalize", help="Run Phase 4 deterministic pipeline")
finalize.add_argument("project", help="Project slug or path")
finalize.add_argument("--input", default="phase4/final_zh.md", help="Chinese Markdown source for default v0.20 finalization")
finalize.add_argument("--legacy-translate", action="store_true", help="Use legacy final_en -> translate -> polish pipeline")
finalize.add_argument("--polish", action="store_true", help="Run optional Chinese polish step before rendering")
finalize.add_argument("--report-engine", choices=["reportlab", "quarto"], default=None)
finalize.add_argument("--no-docx", action="store_true")
finalize.add_argument("--no-pdf", action="store_true")
finalize.add_argument("--translate-workers", type=int, default=0)
finalize.add_argument("--glossary-workers", type=int, default=4)
finalize.add_argument("--polish-workers", type=int, default=0)
@@ -295,6 +769,7 @@ def build_parser() -> argparse.ArgumentParser:
metavar="ROLE=MODEL",
help="Override one role model, repeatable",
)
models.add_argument("--probe", action="store_true", help="Send tiny health checks to resolved role models")
models.add_argument("--json", action="store_true", help="Emit JSON")
models.set_defaults(func=cmd_models)
+29 -42
View File
@@ -1,61 +1,48 @@
#!/usr/bin/env python3
"""Install the Codex native adapter files into hidden project directories.
"""Backward-compatible wrapper for deploying the Codex adapter.
The Codex desktop sandbox may block agent-created writes into `.codex` and
`.agents/skills`. Run this script locally from the repository root when that
happens.
v0.20 keeps Codex adapter templates in the repository, but deploys the usable
adapter files to a Codex home outside the checkout.
"""
from __future__ import annotations
import argparse
import shutil
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
TEMPLATE_ROOT = REPO_ROOT / "codex_adapter_templates" / "codex"
CODEX_ROOT = REPO_ROOT / ".codex"
AGENTS_SKILLS = REPO_ROOT / ".agents" / "skills"
OPENCODE_SKILLS = REPO_ROOT / ".opencode" / "skills"
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
def copy_tree_contents(src: Path, dst: Path, *, force: bool) -> list[Path]:
written: list[Path] = []
if not src.exists():
raise SystemExit(f"template source not found: {src}")
dst.mkdir(parents=True, exist_ok=True)
for item in src.rglob("*"):
rel = item.relative_to(src)
target = dst / rel
if item.is_dir():
target.mkdir(parents=True, exist_ok=True)
continue
if target.exists() and not force:
continue
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(item, target)
written.append(target)
return written
from scripts.deploy_adapters import deploy_codex, print_result
def main() -> int:
parser = argparse.ArgumentParser(description="Install Codex native adapter")
parser.add_argument("--force", action="store_true", help="overwrite existing adapter files")
parser.add_argument("--skip-skills", action="store_true", help="do not copy .opencode/skills to .agents/skills")
parser = argparse.ArgumentParser(description="Deploy Codex native adapter")
parser.add_argument("--target", type=Path, help="Codex home target; defaults to $CODEX_HOME or ~/.codex")
parser.add_argument("--force", action="store_true", help="overwrite existing files and create .bak backups")
parser.add_argument("--skip-skills", action="store_true", help="do not copy canonical skills into target/skills")
parser.add_argument("--include-config", action="store_true", help="also copy config.toml; off by default to avoid overwriting global Codex config")
parser.add_argument("--dry-run", action="store_true", help="show files that would be written")
args = parser.parse_args()
codex_written = copy_tree_contents(TEMPLATE_ROOT, CODEX_ROOT, force=args.force)
skills_written: list[Path] = []
if not args.skip_skills:
skills_written = copy_tree_contents(OPENCODE_SKILLS, AGENTS_SKILLS, force=args.force)
print("Codex adapter installed.")
print(f" .codex files written: {len(codex_written)}")
print(f" .agents skills files written: {len(skills_written)}")
if codex_written:
for path in codex_written:
print(f" {path.relative_to(REPO_ROOT)}")
result = deploy_codex(
target=args.target,
force=args.force,
skip_skills=args.skip_skills,
include_config=args.include_config,
dry_run=args.dry_run,
)
print_result(result)
print()
print("Note: this no longer writes repository-local .codex files by default.")
print("Run Codex from this repository after deployment:")
if args.include_config:
print(" codex --profile deep-research")
else:
print(" codex")
print("Note: config.toml is not copied by default. Use --include-config only if you want the bundled profile.")
return 0
+5 -2
View File
@@ -47,7 +47,9 @@ def resolve_model_profile(
if selected not in profiles:
raise ModelConfigError(f"unknown model profile: {selected}")
roles = dict((profiles[selected] or {}).get("roles") or {})
selected_profile = profiles[selected] or {}
roles = dict(selected_profile.get("roles") or {})
task_types = dict(selected_profile.get("task_types") or defaults.get("task_types") or {})
if defaults.get("script_models"):
for role, model in (defaults.get("script_models") or {}).items():
roles.setdefault(role, model)
@@ -56,8 +58,9 @@ def resolve_model_profile(
return {
"profile": selected,
"description": (profiles[selected] or {}).get("description", ""),
"description": selected_profile.get("description", ""),
"roles": roles,
"task_types": task_types,
}
+42 -8
View File
@@ -29,6 +29,35 @@ MAX_RETRIES = 5
RETRYABLE_STATUSES = {408, 429, 500, 502, 503, 504, 520, 524}
_ANTHROPIC_MODEL_ALIASES = {
"anthropic/claude-opus-4-7": "anthropic/claude-opus-4.7",
"anthropic/claude-opus-4-6": "anthropic/claude-opus-4.6",
"anthropic/claude-opus-4-5": "anthropic/claude-opus-4.5",
"anthropic/claude-opus-4-1": "anthropic/claude-opus-4.1",
"anthropic/claude-sonnet-4-6": "anthropic/claude-sonnet-4.6",
"anthropic/claude-sonnet-4-5": "anthropic/claude-sonnet-4.5",
"anthropic/claude-haiku-4-5": "anthropic/claude-haiku-4.5",
}
_MODELS_WITHOUT_TEMPERATURE = {
"anthropic/claude-opus-4.7",
}
def normalize_zenmux_model(model: str) -> str:
"""Convert adapter-facing model IDs to ZenMux OpenAI API model IDs."""
normalized = model.strip()
if normalized.startswith("zenmux-anthropic/"):
normalized = "anthropic/" + normalized.removeprefix("zenmux-anthropic/")
elif normalized.startswith("zenmux/"):
normalized = normalized.removeprefix("zenmux/")
return _ANTHROPIC_MODEL_ALIASES.get(normalized, normalized)
def model_accepts_temperature(model: str) -> bool:
"""Return whether the ZenMux API accepts `temperature` for this model."""
return normalize_zenmux_model(model) not in _MODELS_WITHOUT_TEMPERATURE
@dataclass
class UsageStats:
"""聚合一次脚本运行的 token 消耗。"""
@@ -163,12 +192,14 @@ class ZenMuxClient:
messages.extend(extra_messages)
messages.append({"role": "user", "content": user})
api_model = normalize_zenmux_model(model)
body: dict[str, Any] = {
"model": model,
"model": api_model,
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens,
}
if model_accepts_temperature(api_model):
body["temperature"] = temperature
if web_search:
body["web_search_options"] = web_search_options or {}
headers = {
@@ -200,14 +231,14 @@ class ZenMuxClient:
raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
usage = data.get("usage", {}) or {}
with self._usage_lock:
self.usage.add(model, usage)
self.usage.add(api_model, usage)
content = ""
choices = data.get("choices") or []
if choices:
msg = choices[0].get("message") or {}
content = msg.get("content") or ""
self._log({
"tag": tag, "model": model, "attempt": attempt,
"tag": tag, "model": api_model, "requested_model": model, "attempt": attempt,
"elapsed": round(elapsed, 2),
"usage": usage,
"out_chars": len(content),
@@ -256,12 +287,14 @@ class ZenMuxClient:
messages.extend(extra_messages)
messages.append({"role": "user", "content": user})
api_model = normalize_zenmux_model(model)
body: dict[str, Any] = {
"model": model,
"model": api_model,
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens,
}
if model_accepts_temperature(api_model):
body["temperature"] = temperature
if web_search:
body["web_search_options"] = web_search_options or {}
@@ -309,7 +342,7 @@ class ZenMuxClient:
usage = data.get("usage", {}) or {}
with self._usage_lock:
self.usage.add(model, usage)
self.usage.add(api_model, usage)
message = ((data.get("choices") or [{}])[0].get("message") or {})
content = message.get("content") or ""
@@ -324,7 +357,8 @@ class ZenMuxClient:
urls.append(url_item)
self._log({
"tag": tag,
"model": model,
"model": api_model,
"requested_model": model,
"attempt": attempt,
"elapsed": round(elapsed, 2),
"usage": usage,
+2
View File
@@ -0,0 +1,2 @@
"""Report rendering helpers for the v0.20 Python core."""
+32
View File
@@ -0,0 +1,32 @@
"""Font resolution helpers for PDF rendering."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
@dataclass(frozen=True)
class QuartoFonts:
main_font: str
sans_font: str
requires_system_fonts: bool
def resolve_quarto_fonts(fonts_dir: Path) -> QuartoFonts:
"""Resolve Quarto font names.
Quarto/xelatex currently uses installed font family names. We still accept
fonts_dir so callers can validate/report environment state consistently.
"""
expected = [
fonts_dir / "SourceHanSerifSC-Regular.otf",
fonts_dir / "SourceHanSansSC-Bold.otf",
]
requires_system_fonts = not all(path.exists() for path in expected)
return QuartoFonts(
main_font="Source Han Serif CN",
sans_font="Source Han Sans CN",
requires_system_fonts=requires_system_fonts,
)
+61
View File
@@ -0,0 +1,61 @@
"""Reference-list generation from Deep Research sources.jsonl."""
from __future__ import annotations
import json
import re
from pathlib import Path
def cited_source_keys(md_text: str) -> set[str]:
return set(re.findall(r"\[src_([a-zA-Z0-9_-]+)\]", md_text))
def build_references_block(sources_path: Path | None, md_text: str) -> str:
"""Build a compact references section for actually cited src IDs."""
if not sources_path or not sources_path.exists():
return "(参考文献列表:sources.jsonl 未找到)"
cited = cited_source_keys(md_text)
if not cited:
return ""
sources: dict[str, dict] = {}
with sources_path.open(encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
sid = obj.get("id", "")
key = sid.replace("src_", "")
if key in cited:
sources[sid] = obj
if not sources:
return ""
lines = ["## 参考文献\n"]
for sid in sorted(sources.keys()):
s = sources[sid]
authors = ", ".join(s.get("authors", [])) if s.get("authors") else ""
year = s.get("year", "")
title = s.get("title", sid)
venue = s.get("venue", "")
url = s.get("url", "")
entry = f"- **[{sid}]** "
if authors:
entry += f"{authors}. "
if year:
entry += f"({year}). "
entry += f"*{title}*"
if venue:
entry += f". {venue}"
if url:
entry += f". <{url}>"
lines.append(entry)
return "\n".join(lines)
+6
View File
@@ -0,0 +1,6 @@
"""v0.20 Python runtime core for Deep Research.
The runtime layer is intentionally platform-neutral: OpenCode, Codex, and
Claude Code should call into these modules instead of owning orchestration.
"""
+46
View File
@@ -0,0 +1,46 @@
"""Project artifact helpers shared by the Python runtime."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[2]
PROJECTS_DIR = REPO_ROOT / "projects"
def resolve_project(project: str | Path) -> Path:
p = Path(project)
if p.is_dir():
return p.resolve()
candidate = PROJECTS_DIR / str(project)
if candidate.is_dir():
return candidate.resolve()
raise FileNotFoundError(f"project not found: {project}")
def load_manifest(project_root: Path) -> dict[str, Any]:
path = project_root / "manifest.json"
if not path.exists():
raise FileNotFoundError(f"manifest not found: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def write_manifest(project_root: Path, manifest: dict[str, Any]) -> None:
path = project_root / "manifest.json"
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def ensure_phase_dirs(project_root: Path) -> None:
for rel in (
"phase1",
"phase2/drafts",
"phase2/evidence",
"phase2/packets",
"phase3",
"phase4",
):
(project_root / rel).mkdir(parents=True, exist_ok=True)
+202
View File
@@ -0,0 +1,202 @@
"""Chapter brief aggregation and Chinese chapter assembly."""
from __future__ import annotations
import json
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Callable
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
from scripts.runtime.skills import SkillRegistry
from scripts.runtime.tasks import load_task_cards, validate_packet
from scripts.runtime.workers import ChatClient
def validate_chapter_brief(brief: dict) -> None:
required = {
"chapter_id",
"chapter_title",
"packet_ids",
"core_claims",
"evidence_items",
"counter_evidence",
"source_ids",
"open_questions",
"assembly_notes",
}
missing = sorted(required - set(brief))
if missing:
raise ValueError(f"chapter brief missing fields: {missing}")
if not brief["chapter_id"]:
raise ValueError("chapter_id required")
if not brief["packet_ids"]:
raise ValueError("chapter brief requires at least one packet")
if not brief["core_claims"]:
raise ValueError("chapter brief requires core_claims")
if not brief["evidence_items"]:
raise ValueError("chapter brief requires evidence_items")
if not brief["counter_evidence"]:
raise ValueError("chapter brief requires counter_evidence")
def validate_chapter_markdown_citations(markdown: str, brief: dict) -> None:
validate_chapter_brief(brief)
cited = set(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", markdown))
allowed = set(brief.get("source_ids") or [])
unknown = sorted(cited - allowed)
if unknown:
raise ValueError(f"unknown citation ids in {brief['chapter_id']}: {unknown}")
def _chapter_title_from_id(chapter_id: str) -> str:
try:
index = int(chapter_id.replace("ch", ""))
return f"{index}"
except ValueError:
return chapter_id
def build_chapter_briefs(project_root: Path) -> list[dict]:
cards = load_task_cards(project_root / "phase2" / "task_cards.json")
grouped: dict[str, list[tuple[str, dict]]] = {}
for card in cards:
packet_path = project_root / card.output_packet
if not packet_path.exists():
continue
packet = json.loads(packet_path.read_text(encoding="utf-8"))
validate_packet(packet)
for chapter_id in card.chapter_ids:
grouped.setdefault(chapter_id, []).append((card.task_id, packet))
briefs: list[dict] = []
out_dir = project_root / "phase2" / "chapter_briefs"
out_dir.mkdir(parents=True, exist_ok=True)
for chapter_id in sorted(grouped):
packet_pairs = sorted(grouped[chapter_id], key=lambda item: item[0])
packet_ids = [item[0] for item in packet_pairs]
packets = [item[1] for item in packet_pairs]
source_ids = sorted({sid for packet in packets for sid in packet.get("source_ids", [])})
brief = {
"chapter_id": chapter_id,
"chapter_title": _chapter_title_from_id(chapter_id),
"packet_ids": packet_ids,
"core_claims": [claim for packet in packets for claim in packet.get("claims", [])],
"evidence_items": [item for packet in packets for item in packet.get("evidence_items", [])],
"counter_evidence": [item for packet in packets for item in packet.get("counter_evidence", [])],
"source_ids": source_ids,
"open_questions": [q for packet in packets for q in packet.get("open_questions", [])],
"assembly_notes": [
"用中文写正式章节,英文仅保留在必要的来源标题、原文摘录、DOI/URL 中。",
"避免碎片化:不要按 packet 逐段堆砌,要先提炼本章主线,再组织证据。",
"每个事实、数字和关键判断都必须保留 [src_xxx] 引用。",
"必须纳入 counter_evidence,并说明它如何影响结论置信度。",
],
}
validate_chapter_brief(brief)
(out_dir / f"{chapter_id}.json").write_text(
json.dumps(brief, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
briefs.append(brief)
return briefs
def build_chapter_user_prompt(brief: dict) -> str:
return (
"请根据以下 chapter brief 写一章正式中文 Markdown 正文。\n"
"目标是形成一个完整章节,而不是 packet 摘要。避免碎片化,按金字塔结构组织:章首先给结论,再用证据支撑。\n"
"要求:标题必须是观点型判断;每个数字和事实保留 [src_xxx];纳入反方证据;不要出现调度元数据。\n"
"禁止写空泛咨询腔。每个二级小节都必须至少落下 2 个具体审计发现、法规要求、SOP/记录/参数/现场观察或整改证据;不要只写原则。\n"
"正文末尾必须增加“证据落点与待补证据”小节,用表格列出:关键判断、已使用证据 source_id、已落地整改动作、仍缺证据。若证据不足,直接标注需回炉 Phase 2,不要用泛泛表述补齐。\n"
"只输出 Markdown,不要输出解释。\n\n"
f"{json.dumps(brief, ensure_ascii=False, indent=2)}"
)
class ChapterAssemblyWorker:
def __init__(
self,
*,
role: RoleDefinition,
client: ChatClient,
skill_registry: SkillRegistry | None = None,
) -> None:
self.role = role
self.client = client
self.skill_registry = skill_registry or SkillRegistry()
def _system_prompt(self) -> str:
skill_texts = []
for name in self.role.skills:
try:
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
except FileNotFoundError:
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
return (
"你是 Deep Research v0.20 的中文章节组装 worker。\n"
"你的职责是把结构化证据包收束成连贯章节,解决并发研究造成的碎片化。\n"
"不得编造来源,不得删除关键反方证据。\n\n"
+ "\n\n".join(skill_texts)
)
def write_chapter(self, *, project_root: Path, brief: dict) -> Path:
validate_chapter_brief(brief)
markdown = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_chapter_user_prompt(brief),
temperature=self.role.temperature,
max_tokens=self.role.max_tokens,
tag=f"chapter:{brief['chapter_id']}",
)
validate_chapter_markdown_citations(markdown, brief)
out = project_root / "phase2" / "drafts" / f"{brief['chapter_id']}.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(markdown.rstrip() + "\n", encoding="utf-8")
return out
def _write_chapter_error(project_root: Path, brief: dict, error: Exception) -> None:
path = project_root / "phase2" / "chapter_errors" / f"{brief.get('chapter_id', 'unknown')}.json"
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"chapter_id": brief.get("chapter_id"),
"status": "failed",
"error": str(error),
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def run_chapter_assembly_workers(
*,
project_root: Path,
briefs: list[dict],
runtime: RuntimeProfile,
client_factory: Callable[[RoleDefinition], ChatClient],
workers: int,
) -> int:
role = runtime.role_for_task("chapter_assembly")
max_workers = max(1, min(workers, role.max_concurrency))
def run_one(brief: dict) -> tuple[dict, Path | None, Exception | None]:
try:
worker = ChapterAssemblyWorker(role=role, client=client_factory(role))
return brief, worker.write_chapter(project_root=project_root, brief=brief), None
except Exception as error:
return brief, None, error
written = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = [pool.submit(run_one, brief) for brief in briefs]
for future in as_completed(futures):
brief, path, error = future.result()
if error is not None:
_write_chapter_error(project_root, brief, error)
continue
if path is None:
_write_chapter_error(project_root, brief, RuntimeError("chapter worker returned no output path"))
continue
written += 1
return written
+229
View File
@@ -0,0 +1,229 @@
"""Phase 0 user-provided material ingestion."""
from __future__ import annotations
import base64
import os
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import requests
DEFAULT_FIRERED_OCR_ENDPOINT = "http://192.168.50.100:8001"
DEFAULT_OCR_MAX_PAGES = 50
@dataclass(frozen=True)
class OcrResult:
text: str
pages_processed: int
output_path: Path
def safe_filename(path: Path) -> str:
name = path.name.strip()
return name or "material"
def extract_pdf_text(path: Path) -> tuple[str, int, bool]:
from pypdf import PdfReader
reader = PdfReader(str(path))
chunks: list[str] = []
for index, page in enumerate(reader.pages, start=1):
text = (page.extract_text() or "").strip()
if text:
chunks.append(f"\n\n## Page {index}\n\n{text}")
combined = "".join(chunks).strip()
ocr_required = len(combined) < max(20, len(reader.pages) * 20)
return combined, len(reader.pages), ocr_required
def ocr_endpoint_from_env() -> str:
return os.environ.get("DEEP_RESEARCH_OCR_ENDPOINT", DEFAULT_FIRERED_OCR_ENDPOINT).rstrip("/")
def ocr_max_pages_from_env() -> int:
raw = os.environ.get("DEEP_RESEARCH_OCR_MAX_PAGES")
if not raw:
return DEFAULT_OCR_MAX_PAGES
try:
return max(1, int(raw))
except ValueError:
return DEFAULT_OCR_MAX_PAGES
def render_pdf_pages(pdf_path: Path, output_dir: Path, *, max_pages: int) -> list[Path]:
import fitz
pages_dir = output_dir / f"{pdf_path.stem}.ocr-pages"
pages_dir.mkdir(parents=True, exist_ok=True)
image_paths: list[Path] = []
doc = fitz.open(pdf_path)
try:
for index, page in enumerate(doc[:max_pages], start=1):
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
image_path = pages_dir / f"page-{index:03d}.png"
pix.save(image_path)
image_paths.append(image_path)
finally:
doc.close()
return image_paths
def data_url_for_image(path: Path) -> str:
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
return f"data:image/png;base64,{encoded}"
def call_firered_ocr(image_path: Path, *, endpoint: str) -> str:
payload = {
"model": "firered-ocr",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "请识别图片中的全部文字,保持原有顺序,只输出文字。"},
{"type": "image_url", "image_url": {"url": data_url_for_image(image_path)}},
],
}
],
"temperature": 0,
"max_tokens": 3000,
}
response = requests.post(f"{endpoint.rstrip('/')}/v1/chat/completions", json=payload, timeout=60)
if not response.ok:
raise RuntimeError(f"{response.status_code} {response.text[:500]}")
data = response.json()
return str(data["choices"][0]["message"].get("content") or "").strip()
def ocr_pdf_with_firered(*, pdf_path: Path, output_dir: Path, endpoint: str, max_pages: int) -> OcrResult:
image_paths = render_pdf_pages(pdf_path, output_dir, max_pages=max_pages)
chunks: list[str] = []
for index, image_path in enumerate(image_paths, start=1):
text = call_firered_ocr(image_path, endpoint=endpoint)
if text:
chunks.append(f"\n\n## OCR Page {index}\n\n{text}")
combined = "".join(chunks).strip()
output_path = output_dir / f"{pdf_path.stem}.ocr.md"
body = [
f"# OCR Material: {pdf_path.name}",
"",
f"- source_path: {pdf_path}",
f"- endpoint: {endpoint}",
f"- pages_processed: {len(image_paths)}",
"",
combined or "OCR 未返回可用文本。",
"",
]
output_path.write_text("\n".join(body), encoding="utf-8")
return OcrResult(text=combined, pages_processed=len(image_paths), output_path=output_path)
def ingest_input_materials(project_root: Path, materials: list[str] | None) -> list[dict[str, Any]]:
inventory: list[dict[str, Any]] = []
if not materials:
return inventory
inputs_dir = project_root / "phase0" / "inputs"
extracted_dir = project_root / "phase0" / "extracted"
ocr_endpoint = ocr_endpoint_from_env()
ocr_max_pages = ocr_max_pages_from_env()
inputs_dir.mkdir(parents=True, exist_ok=True)
extracted_dir.mkdir(parents=True, exist_ok=True)
for raw in materials:
source = Path(raw).expanduser()
if not source.exists():
inventory.append({"kind": "note", "note": raw})
continue
copied = inputs_dir / safe_filename(source)
shutil.copy2(source, copied)
item: dict[str, Any] = {
"kind": source.suffix.lower().lstrip(".") or "file",
"source_path": str(source),
"copied_to": str(copied.relative_to(project_root)),
"size_bytes": source.stat().st_size,
}
if source.suffix.lower() == ".pdf":
text, pages, ocr_required = extract_pdf_text(source)
extracted = extracted_dir / f"{source.stem}.md"
ocr_result: OcrResult | None = None
ocr_error: str | None = None
if ocr_required:
try:
ocr_result = ocr_pdf_with_firered(
pdf_path=source,
output_dir=extracted_dir,
endpoint=ocr_endpoint,
max_pages=min(pages, ocr_max_pages),
)
if ocr_result.text:
text = "\n\n".join(part for part in [text, ocr_result.text] if part)
except Exception as exc: # noqa: BLE001 - ingestion should not block project init.
ocr_error = str(exc)
body = [
f"# Extracted Material: {source.name}",
"",
f"- source_path: {source}",
f"- copied_to: {copied.relative_to(project_root)}",
f"- pages: {pages}",
f"- ocr_required: {str(ocr_required).lower()}",
f"- ocr_status: {'completed' if ocr_result else 'failed' if ocr_error else 'not_required'}",
"",
text or "未能从 PDF 直接抽取文本;该材料可能需要 OCR。",
"",
]
if ocr_error:
body.extend(["## OCR Error", "", ocr_error, ""])
extracted.write_text("\n".join(body), encoding="utf-8")
item.update(
{
"pages": pages,
"extracted_to": str(extracted.relative_to(project_root)),
"text_chars": len(text),
"ocr_required": ocr_required,
"ocr_status": "completed" if ocr_result else "failed" if ocr_error else "not_required",
}
)
if ocr_result:
item.update(
{
"ocr_endpoint": ocr_endpoint,
"ocr_pages_processed": ocr_result.pages_processed,
"ocr_extracted_to": str(ocr_result.output_path.relative_to(project_root)),
"ocr_text_chars": len(ocr_result.text),
}
)
if ocr_error:
item["ocr_error"] = ocr_error
else:
item["ocr_required"] = source.suffix.lower() in {".png", ".jpg", ".jpeg", ".tif", ".tiff"}
inventory.append(item)
return inventory
def render_material_inventory(inventory: list[dict[str, Any]]) -> str:
if not inventory:
return "- 暂无;可通过 `--input-material` 加入审计报告、问题清单或内部记录。"
lines: list[str] = []
for item in inventory:
if item.get("kind") == "note":
lines.append(f"- 备注:{item.get('note', '')}")
continue
marker = ";需要 OCR" if item.get("ocr_required") else ""
ocr = f"OCR{item.get('ocr_status')}" if item.get("ocr_status") else ""
extracted = item.get("extracted_to")
extra = f";抽取文本:{extracted}" if extracted else ""
lines.append(
f"- {item.get('copied_to')}{item.get('kind')}{item.get('size_bytes', 0)} bytes{extra}{marker}{ocr}"
)
return "\n".join(lines)
+60
View File
@@ -0,0 +1,60 @@
"""Research method registry for Phase 1 framework selection."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_METHOD_CONFIG = REPO_ROOT / "configs" / "research_methods.yaml"
@dataclass(frozen=True)
class ResearchMethod:
key: str
name: str
best_for: list[str]
structure_principle: str
task_axes: list[str]
framework_sections: list[str]
class ResearchMethodRegistry:
def __init__(self, path: Path | None = None) -> None:
self.path = path or DEFAULT_METHOD_CONFIG
self._data = self._load()
def _load(self) -> dict[str, Any]:
if not self.path.exists():
raise FileNotFoundError(f"research method config not found: {self.path}")
data = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {}
if not isinstance(data, dict) or "methods" not in data:
raise ValueError(f"invalid research method config: {self.path}")
return data
@property
def default_method(self) -> str:
return (self._data.get("defaults") or {}).get("method", "mckinsey_market")
def list_names(self) -> list[str]:
return sorted((self._data.get("methods") or {}).keys())
def get(self, key: str | None = None) -> ResearchMethod:
selected = key or self.default_method
methods = self._data.get("methods") or {}
if selected not in methods:
raise KeyError(f"unknown research_method: {selected}")
item = methods[selected] or {}
return ResearchMethod(
key=selected,
name=item.get("name", selected),
best_for=list(item.get("best_for") or []),
structure_principle=item.get("structure_principle", ""),
task_axes=list(item.get("task_axes") or []),
framework_sections=list(item.get("framework_sections") or []),
)
+79
View File
@@ -0,0 +1,79 @@
"""Deterministic orchestration helpers for v0.20."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from scripts.runtime.artifacts import ensure_phase_dirs, load_manifest, write_manifest
from scripts.runtime.methods import ResearchMethodRegistry
from scripts.runtime.tasks import generate_task_cards, write_task_cards
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def create_phase2_task_cards(
project_root: Path,
*,
axes: list[str] | None = None,
dry_run: bool = False,
) -> list[dict[str, object]]:
framework = project_root / "phase1" / "framework.md"
if not framework.exists():
raise FileNotFoundError(f"framework not found: {framework}")
method_key = None
if (project_root / "manifest.json").exists():
method_key = load_manifest(project_root).get("research_method")
method = ResearchMethodRegistry().get(method_key)
cards = generate_task_cards(project_root.name, framework.read_text(encoding="utf-8"), axes=axes, method=method)
if not dry_run:
ensure_phase_dirs(project_root)
write_task_cards(project_root / "phase2" / "task_cards.json", cards)
manifest = load_manifest(project_root)
phase2 = manifest.setdefault("phase2", {})
phase2.update(
{
"status": "in_progress",
"runtime": "python-core-v0.20",
"research_method": method.key,
"task_cards_path": "phase2/task_cards.json",
"task_cards_total": len(cards),
"updated_at": utc_now_iso(),
}
)
write_manifest(project_root, manifest)
return [card.to_dict() for card in cards]
def write_placeholder_packets(
project_root: Path,
task_cards: list[dict[str, object]],
*,
dry_run: bool = False,
) -> int:
"""Create packet skeletons for manual/API completion.
This keeps the first v0.20 implementation deterministic and resumable; LLM
calls can later fill the same schema without changing downstream readers.
"""
count = 0
for card in task_cards:
packet_path = project_root / str(card["output_packet"])
packet = {
"task_id": card["task_id"],
"claims": [],
"evidence_items": [],
"counter_evidence": [],
"source_ids": [],
"source_quality_notes": [],
"open_questions": ["待由 Python role worker 调用模型补全。"],
"raw_quotes_or_notes": [],
}
if not dry_run:
packet_path.parent.mkdir(parents=True, exist_ok=True)
packet_path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
count += 1
return count
+341
View File
@@ -0,0 +1,341 @@
"""Phase 1 project initialization and framework generation."""
from __future__ import annotations
import hashlib
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from scripts.runtime.artifacts import PROJECTS_DIR, ensure_phase_dirs, load_manifest, write_manifest
from scripts.runtime.materials import ingest_input_materials, render_material_inventory
from scripts.runtime.methods import ResearchMethod, ResearchMethodRegistry
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def slugify_topic(topic: str) -> str:
slug = re.sub(r"[^a-zA-Z0-9]+", "-", topic.lower()).strip("-")
if slug:
return slug[:80]
digest = hashlib.sha1(topic.encode("utf-8")).hexdigest()[:8]
return f"research-{digest}"
def create_project(
*,
topic: str,
slug: str | None = None,
projects_dir: Path = PROJECTS_DIR,
method_key: str | None = None,
report_type: str = "research",
model_profile: str = "medium",
target_words: int = 30000,
input_materials: list[str] | None = None,
) -> Path:
method = ResearchMethodRegistry().get(method_key)
project_slug = slug or slugify_topic(topic)
project_root = projects_dir / project_slug
if project_root.exists():
raise FileExistsError(f"project already exists: {project_root}")
ensure_phase_dirs(project_root)
(project_root / "phase0" / "inputs").mkdir(parents=True, exist_ok=True)
(project_root / "phase0" / "extracted").mkdir(parents=True, exist_ok=True)
material_inventory = ingest_input_materials(project_root, input_materials)
now = utc_now_iso()
manifest: dict[str, Any] = {
"version": "0.20.0",
"runtime": "python-core-v0.20",
"topic": topic,
"slug": project_slug,
"report_title": topic,
"type": report_type,
"work_language": "zh",
"model_profile": model_profile,
"research_method": method.key,
"target_words": target_words,
"input_materials": input_materials or [],
"material_inventory": material_inventory,
"created_at": now,
"updated_at": now,
"phase1": {
"status": "initialized",
"approved": False,
"requires_user_interview": True,
"material_brief_path": "phase1/material_brief.md",
},
"phase2": {"status": "pending"},
"phase3": {"status": "pending"},
"phase4": {"status": "pending"},
}
write_manifest(project_root, manifest)
_write_interview_seed(project_root, manifest, method)
write_material_brief(project_root, manifest, method)
return project_root
def _write_interview_seed(project_root: Path, manifest: dict[str, Any], method: ResearchMethod) -> None:
material_text = render_material_inventory(manifest.get("material_inventory") or [])
text = (
f"# Phase 1 访谈记录\n\n"
f"- 主题:{manifest['topic']}\n"
f"- 研究方法:{method.key} - {method.name}\n"
f"- 报告类型:{manifest['type']}\n"
f"- 目标字数:{manifest['target_words']}\n"
f"- 工作语言:中文主写作;检索关键词、证据摘录和来源笔记可保留英文。\n\n"
f"## 已提供材料\n\n{material_text}\n\n"
"## 后续访谈问题\n\n"
"1. 本报告最重要的决策用途是什么?\n"
"2. 是否有必须覆盖或必须排除的公司、产品、工艺、市场或法规范围?\n"
"3. 结论偏好是战略建议、风险清单、投资判断,还是执行路线图?\n"
)
(project_root / "phase1" / "interview.md").write_text(text, encoding="utf-8")
def _material_excerpt(project_root: Path, rel_path: str, *, max_chars: int = 1200) -> str:
path = project_root / rel_path
if not path.exists():
return "(未找到抽取文本)"
text = path.read_text(encoding="utf-8")
compact = "\n".join(line.rstrip() for line in text.splitlines() if line.strip())
return compact[:max_chars] + ("..." if len(compact) > max_chars else "")
def _derive_material_observations(project_root: Path, inventory: list[dict[str, Any]]) -> list[str]:
combined_parts: list[str] = []
for item in inventory:
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
if rel and (project_root / rel).exists():
combined_parts.append((project_root / rel).read_text(encoding="utf-8"))
text = "\n".join(combined_parts)
checks = [
("审计范围覆盖生产管理、原液、制剂和无菌相关模块,报告需要同时处理 GMP 合规、工艺转移和运营协同,而不是只写质量体系。", ["生产管理", "原液", "制剂", "无菌"]),
("材料显示高风险项为 0、中风险项为 1,适合采用“商业化 readiness 与系统成熟度差距”而非“体系失控”作为初始假设。", ["高风险 0", "中风险1", "低风险7"]),
("商业化经验、无菌保障细节、文件要求与执行一致性是需要访谈确认的主线风险。", ["商业化经验不足", "无菌保障", "文件要求与执行一致性"]),
("工艺规程、批记录、CPP/CQA、VMPR/VMP、验证主计划等内容反复出现,说明工艺验证和商业化文件体系可能是 Phase 2 的重点证据轴。", ["CPP", "CQA", "VMPR", "VMP"]),
("温度、压差、WFI、冷却段微生物、RABS/ORABS、first air、APS 等无菌和设施细节需要映射到 EU Annex 1、NMPA GMP 和企业 SOP。", ["温度", "压差", "WFI", "APS"]),
("复盘材料包含责任人和局部答复,后续整改路线图应尽量回填 owner、期限、关闭证据和复核机制。", ["填写人", "是否已经回答完整", "整改"]),
]
observations = [message for message, needles in checks if any(needle in text for needle in needles)]
return observations or ["材料已导入但尚未形成足够结构化判断;需要先访谈确认研究用途、范围和优先级。"]
def write_material_brief(
project_root: Path,
manifest: dict[str, Any] | None = None,
method: ResearchMethod | None = None,
) -> Path:
"""Write a Phase 0/1 material brief that must be reviewed before Phase 2."""
manifest = manifest or load_manifest(project_root)
method = method or ResearchMethodRegistry().get(manifest.get("research_method"))
inventory = manifest.get("material_inventory") or []
lines = [
f"# Phase 0 材料简报:{manifest.get('topic', project_root.name)}",
"",
"status: 待用户确认",
f"research_method: {method.key}",
"",
"## 已导入材料",
"",
render_material_inventory(inventory),
"",
"## 材料初步解读",
"",
"以下内容由 Python core 从已落盘材料抽样生成,只作为访谈起点;不得直接视为最终结论。",
"",
]
lines.extend(["## 初步问题聚类(待访谈确认)", ""])
for observation in _derive_material_observations(project_root, inventory):
lines.append(f"- {observation}")
lines.append("")
lines.append("## 材料摘录")
lines.append("")
for item in inventory:
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
if not rel:
continue
lines.extend(
[
f"### {Path(rel).name}",
"",
_material_excerpt(project_root, rel),
"",
]
)
lines.extend(
[
"## 建议访谈确认点",
"",
"1. 本报告的最重要用途是什么:内部整改、客户沟通、董事会决策,还是外部审计准备?",
"2. 哪些审计发现最需要优先展开:无菌保障、工艺验证、数据完整性、质量体系闭环,还是运营协同?",
"3. 是否存在必须排除或脱敏的项目、人员、客户、产品或工艺信息?",
"4. 短中长期整改的时间边界如何定义,例如 30/90/180 天,还是按临床/商业化里程碑划分?",
"5. 是否需要把 NMPA、FDA、EMA、ICH、WHO 的法规基线分别映射到整改责任人和证据包?",
"",
"## Gate",
"",
"请用户确认本材料简报与访谈问题后,再生成或批准 `phase1/framework.md` 并进入 Phase 2。",
"",
]
)
out = project_root / "phase1" / "material_brief.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
return out
CHAPTER_TEMPLATES: dict[str, list[str]] = {
"mckinsey_market": [
"核心结论先行界定市场机会与约束",
"临床与真实世界证据决定需求天花板",
"监管路径和支付环境重塑商业化节奏",
"竞争格局正在从单点产品转向组合能力",
"专利与技术壁垒决定长期利润池",
"中国市场的准入和供给能力形成独立变量",
"资本市场预期与基本面之间存在可验证偏差",
"反方证据限定结论边界并提示回撤风险",
"战略选择应围绕资源约束排序",
"执行路线图需要把证据缺口转化为行动清单",
],
"gmp_gap_assessment": [
"监管基线决定整改范围而非企业主观偏好",
"现状差距需要按法规条款和业务流程双重定位",
"质量风险分级决定 CAPA 优先级",
"根因分析质量决定整改能否闭环",
"CAPA 设计必须绑定责任人、证据和期限",
"验证计划决定整改是否可被审计接受",
"供应商和外包管理常是系统性缺口放大器",
"数据完整性风险需要独立成章处理",
"实施路线图需要平衡停线风险与合规风险",
"管理层治理机制决定整改能否持续",
],
"cmc_process_risk": [
"工艺流程图是识别放大风险的起点",
"CQA 与 CPP 的映射决定控制策略质量",
"放大过程的失效模式集中在传质、混合和稳定性",
"分析方法和放行标准决定证据可信度",
"技术转移风险来自知识隐性化和现场差异",
"供应链约束会改变工艺控制边界",
"偏差和变更管理决定商业化后的韧性",
"监管沟通策略需要提前固化关键假设",
"反方证据限定平台工艺可复制性",
"CMC 路线图需要把风险转化为验证实验",
],
"rd_go_no_go": [
"科学假设强度决定项目是否值得进入下一阶段",
"POC 证据需要同时证明有效性和可转化性",
"安全性窗口决定适应症与人群选择",
"IP 与 FTO 风险决定商业化自由度",
"开发路径需要把关键不确定性前置验证",
"竞争窗口决定速度是否仍有战略价值",
"CMC 与临床运营能力影响真实可行性",
"反方证据决定 go/no-go 阈值",
"投资强度应与证据成熟度匹配",
"决策门槛需要形成可执行检查表",
],
"management_consulting": [
"现状诊断需要区分症状、根因和约束条件",
"能力差距决定组织改进优先级",
"流程断点揭示跨部门协作成本",
"治理结构决定决策速度和责任清晰度",
"运营模型需要匹配战略目标而非照搬标杆",
"数字化工具只有嵌入流程才产生价值",
"绩效指标需要避免局部最优",
"变革阻力本身是方案设计输入",
"路线图需要把 quick wins 与系统建设分层",
"落地机制决定咨询建议能否转化为成果",
],
"gmp_quality_operations_diagnosis": [
"现场审计发现需要先转化为可验证的系统性问题图谱",
"法规基线决定质量体系差距的严重度与整改边界",
"生产工艺体系风险来自流程、设施、公用系统和验证证据的耦合缺口",
"偏差、变更、CAPA 和数据完整性决定质量系统能否闭环",
"人员能力与质量文化决定制度是否真正落地",
"运营管理问题需要区分组织、流程、会议机制和指标体系缺口",
"跨部门协同断点会放大 GMP 风险和交付风险",
"标杆实践应转化为短中长期整改组合而非口号",
"整改路线图必须绑定责任、优先级、证据和复核机制",
"管理层治理机制决定白帆能否从一次整改转向持续改进",
],
}
def render_framework(project_root: Path, *, method_key: str | None = None, chapter_count: int = 10) -> Path:
manifest = load_manifest(project_root)
registry = ResearchMethodRegistry()
method = registry.get(method_key or manifest.get("research_method"))
if method_key:
manifest["research_method"] = method.key
titles = CHAPTER_TEMPLATES.get(method.key) or CHAPTER_TEMPLATES["mckinsey_market"]
chapter_count = max(8, min(15, chapter_count))
selected = titles[:chapter_count]
quota = max(800, int(manifest.get("target_words", 30000)) // len(selected))
sections = "\n".join(f"- {item}" for item in method.framework_sections)
axes = "".join(method.task_axes)
material_text = render_material_inventory(manifest.get("material_inventory") or [])
lines = [
f"# {manifest.get('report_title') or manifest['topic']}:研究框架",
"",
f"research_method: {method.key}",
f"method_name: {method.name}",
f"work_language: 中文主写作;检索关键词、证据摘录、source title、raw notes 可保留英文。",
f"target_words: {manifest.get('target_words', 30000)}",
"",
"## 方法选择",
"",
f"本项目采用 `{method.key}`,因为其结构原则是:{method.structure_principle}",
"",
"框架模块:",
sections,
"",
"Phase 2 任务轴:",
f"- {axes}",
"",
"## 输入材料与使用边界",
"",
material_text,
"",
"这些材料作为现场问题线索和内部事实起点使用;正式结论仍需结合 NMPA、FDA、EMA、ICH、WHO 等权威法规、指南和最佳实践进行验证。",
"",
"## 中心假设",
"",
f"围绕“{manifest['topic']}”形成可被证据支持或证伪的中文主线;所有核心判断必须绑定来源 ID。",
"",
]
for idx, title in enumerate(selected, start=1):
lines.extend(
[
f"## 第{idx}{title}",
"",
f"建议字数:约 {quota} 字。",
f"研究思路:围绕 `{method.key}` 的方法框架,从 {axes} 等任务轴并发收集 evidence packet,再由 chapter assembly 收束为完整中文章节。",
"证据要求:至少 2 个独立 Tier 1-2 信源;不足时在正文标注待验证;必须包含反方证据。",
"",
]
)
lines.extend(
[
"## 暂停点",
"",
"请先确认 `phase1/material_brief.md` 的材料解读和访谈问题,再确认本框架后进入 Phase 2。若章节逻辑、方法框架或字数配额需要调整,应先修改本文件。",
"",
"确认后运行:`uv run python scripts/dr.py approve <project>`;未批准时 `research` 默认会拒绝推进,可用 `--force` 临时覆盖。",
"",
]
)
out = project_root / "phase1" / "framework.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
manifest["phase1"] = {
"status": "completed",
"approved": False,
"framework_path": "phase1/framework.md",
"research_method": method.key,
"updated_at": utc_now_iso(),
}
manifest["updated_at"] = utc_now_iso()
write_manifest(project_root, manifest)
return out
+157
View File
@@ -0,0 +1,157 @@
"""Deterministic Phase 3 review checks for the Python core."""
from __future__ import annotations
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from scripts.runtime.artifacts import load_manifest, write_manifest
from scripts.runtime.tasks import validate_packet
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def _source_ids_from_jsonl(path: Path) -> set[str]:
ids: set[str] = set()
if not path.exists():
return ids
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
source_id = obj.get("id") or obj.get("source_id")
if source_id:
ids.add(str(source_id))
return ids
def _draft_citations(drafts: list[Path]) -> set[str]:
cited: set[str] = set()
for draft in drafts:
cited.update(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", draft.read_text(encoding="utf-8")))
return cited
def _ready_packet_stems(project_root: Path) -> set[str]:
ready: set[str] = set()
for path in sorted((project_root / "phase2" / "packets").glob("*.json")):
try:
packet = json.loads(path.read_text(encoding="utf-8"))
validate_packet(packet)
except Exception:
continue
ready.add(path.stem)
return ready
def _draft_quality_findings(drafts: list[Path]) -> list[dict[str, Any]]:
findings: list[dict[str, Any]] = []
generic_markers = [
"需要进一步完善",
"应当加强",
"持续改进",
"系统性",
"闭环管理",
"质量文化",
]
for draft in drafts:
text = draft.read_text(encoding="utf-8")
zh_chars = sum(1 for char in text if "\u4e00" <= char <= "\u9fff")
citations = re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", text)
evidence_table_present = "证据落点" in text and "待补证据" in text
if zh_chars >= 1200 and len(set(citations)) < 5:
findings.append({"severity": "P1", "message": f"{draft.name} 引用来源过少,可能未充分使用 evidence packet。"})
if zh_chars >= 1200 and not evidence_table_present:
findings.append({"severity": "P1", "message": f"{draft.name} 缺少“证据落点与待补证据”小节,难以判断 evidence 是否真正落到纸面。"})
generic_count = sum(text.count(marker) for marker in generic_markers)
if zh_chars >= 1200 and generic_count >= 18:
findings.append({"severity": "P1", "message": f"{draft.name} 泛化管理表述过多,需要回炉为具体审计发现、风险影响和整改动作。"})
return findings
def build_phase3_critique(project_root: Path) -> Path:
manifest = load_manifest(project_root)
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
ready_stems = _ready_packet_stems(project_root)
packet_errors = [
path for path in sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
if path.stem not in ready_stems
]
chapter_errors = sorted((project_root / "phase2" / "chapter_errors").glob("*.json"))
sources = _source_ids_from_jsonl(project_root / "phase2" / "sources.jsonl")
cited = _draft_citations(drafts)
missing_sources = sorted(cited - sources) if sources else sorted(cited)
uncited_sources = sorted(sources - cited) if cited else sorted(sources)
findings: list[dict[str, Any]] = []
if not drafts:
findings.append({"severity": "P1", "message": "Phase 2 drafts 缺失,尚不能进入 Phase 4 成稿。"})
if packet_errors:
findings.append({"severity": "P1", "message": f"存在 {len(packet_errors)} 个 packet 失败,需要回炉补证据。"})
if chapter_errors:
findings.append({"severity": "P1", "message": f"存在 {len(chapter_errors)} 个章节组装失败,需要修复引用或重写该章。"})
quality_holds = manifest.get("quality_holds") or []
if quality_holds:
findings.append({"severity": "P1", "message": "存在质量暂停标记:" + ", ".join(quality_holds)})
findings.extend(_draft_quality_findings(drafts))
if missing_sources:
findings.append({"severity": "P1", "message": f"正文引用未在 sources.jsonl 中登记:{', '.join(missing_sources)}"})
if not findings:
findings.append({"severity": "P2", "message": "基础产物完整;仍需人工或大上下文模型审校逻辑链、反方证据和章节叙事。"})
lines = [
"# Phase 3 审校 critique",
"",
f"- 项目:{manifest.get('topic', project_root.name)}",
f"- 运行时:python-core-v0.20",
f"- drafts{len(drafts)}",
f"- packets{len(packets)}",
f"- sources{len(sources)}",
f"- cited_source_ids{', '.join(sorted(cited)) if cited else ''}",
"",
"## Findings",
"",
]
for item in findings:
lines.append(f"- [{item['severity']}] {item['message']}")
lines.extend(
[
"",
"## Residual Risks",
"",
"- 本 deterministic review 只做结构、引用和错误包检查;深层逻辑审校仍建议交给 `phase3_review` 角色执行。",
"- 若 sources 为空,本审校会把所有正文引用视为待登记来源。",
"",
"## Next",
"",
"- 若存在 P1,先回到 Phase 2 修复 packet/chapter 错误。",
"- 若仅有 P2,可进入 `dr.py finalize` 的中文原生成稿路径。",
"",
]
)
out = project_root / "phase3" / "critique.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
phase3 = manifest.setdefault("phase3", {})
phase3.update(
{
"status": "completed",
"critique_path": "phase3/critique.md",
"findings_total": len(findings),
"missing_sources": missing_sources,
"uncited_sources": uncited_sources,
"updated_at": utc_now_iso(),
}
)
manifest["updated_at"] = utc_now_iso()
write_manifest(project_root, manifest)
return out
+111
View File
@@ -0,0 +1,111 @@
"""Runtime role and task-model resolution."""
from __future__ import annotations
from dataclasses import dataclass
from scripts.lib.model_config import resolve_model_profile
ROLE_DEFAULTS = {
"dr_plan": {
"skills": ["document-ingest", "search-gateway", "search-strategy", "source-quality", "length-budget", "mckinsey-method"],
"temperature": 0.4,
"max_tokens": 12000,
"max_concurrency": 1,
},
"dr_pm": {
"skills": ["length-budget", "evidence-table", "mckinsey-method"],
"temperature": 0.2,
"max_tokens": 8000,
"max_concurrency": 1,
},
"dr_searcher": {
"skills": ["search-gateway", "search-strategy", "source-quality"],
"temperature": 0.1,
"max_tokens": 6000,
"max_concurrency": 6,
},
"dr_analyst": {
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table", "mckinsey-method"],
"temperature": 0.3,
"max_tokens": 14000,
"max_concurrency": 6,
},
"dr_verifier": {
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table"],
"temperature": 0.2,
"max_tokens": 10000,
"max_concurrency": 4,
},
"dr_chief_editor": {
"skills": ["mckinsey-method", "evidence-table", "output-hygiene"],
"temperature": 0.2,
"max_tokens": 16000,
"max_concurrency": 1,
},
"dr_editor_in_chief": {
"skills": ["mckinsey-method", "citation-manager", "humanizer-cn", "output-hygiene"],
"temperature": 0.4,
"max_tokens": 20000,
"max_concurrency": 1,
},
"dr_reporter": {
"skills": ["pdf-reportlab", "citation-manager", "output-hygiene"],
"temperature": 0.1,
"max_tokens": 6000,
"max_concurrency": 1,
},
}
@dataclass(frozen=True)
class RoleDefinition:
name: str
model: str
skills: list[str]
temperature: float
max_tokens: int
max_concurrency: int
class RuntimeProfile:
def __init__(self, *, profile: str, roles: dict[str, RoleDefinition], task_types: dict[str, str]) -> None:
self.profile = profile
self.roles = roles
self.task_types = task_types
def role_for_task(self, task_type: str) -> RoleDefinition:
role_name = self.task_types.get(task_type)
if not role_name:
raise KeyError(f"unknown task_type: {task_type}")
if role_name not in self.roles:
raise KeyError(f"task_type {task_type} maps to missing role {role_name}")
return self.roles[role_name]
def resolve_runtime_profile(
*,
profile: str | None = None,
overrides: dict[str, str] | None = None,
) -> RuntimeProfile:
resolved = resolve_model_profile(profile=profile, overrides=overrides)
role_models = resolved["roles"]
roles: dict[str, RoleDefinition] = {}
for name, defaults in ROLE_DEFAULTS.items():
model = role_models.get(name)
if not model:
continue
roles[name] = RoleDefinition(
name=name,
model=model,
skills=list(defaults["skills"]),
temperature=float(defaults["temperature"]),
max_tokens=int(defaults["max_tokens"]),
max_concurrency=int(defaults["max_concurrency"]),
)
return RuntimeProfile(
profile=resolved["profile"],
roles=roles,
task_types=dict(resolved.get("task_types") or {}),
)
+107
View File
@@ -0,0 +1,107 @@
"""Canonical skill registry and adapter sync helpers."""
from __future__ import annotations
import shutil
from dataclasses import dataclass
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
CANONICAL_SKILLS_DIR = REPO_ROOT / ".agents" / "skills"
PROJECT_SKILLS_DIR = REPO_ROOT / "skills"
REQUIRED_SKILLS = {
"search-strategy",
"search-gateway",
"source-quality",
"length-budget",
"evidence-table",
"citation-manager",
"mckinsey-method",
}
@dataclass(frozen=True)
class SkillInfo:
name: str
path: Path
class SkillRegistry:
"""Reads skills from the canonical cross-adapter registry."""
def __init__(self, canonical_dir: Path | None = None) -> None:
self.canonical_dir = canonical_dir or CANONICAL_SKILLS_DIR
def roots(self) -> list[Path]:
roots = [self.canonical_dir]
if self.canonical_dir == CANONICAL_SKILLS_DIR and PROJECT_SKILLS_DIR.exists():
roots.append(PROJECT_SKILLS_DIR)
return roots
def list(self) -> list[SkillInfo]:
seen: set[str] = set()
out: list[SkillInfo] = []
for root in self.roots():
if not root.exists():
continue
for path in sorted(root.glob("*/SKILL.md")):
name = path.parent.name
if name in seen:
continue
seen.add(name)
out.append(SkillInfo(name=name, path=path))
return out
def list_names(self) -> list[str]:
return [item.name for item in self.list()]
def read(self, name: str) -> str:
for root in self.roots():
path = root / name / "SKILL.md"
if path.exists():
return path.read_text(encoding="utf-8")
raise FileNotFoundError(f"skill not found: {name}")
def validate(self, required: set[str] | None = None) -> dict[str, object]:
names = set(self.list_names())
required_names = required or REQUIRED_SKILLS
missing = sorted(required_names - names)
malformed: list[str] = []
for item in self.list():
text = item.path.read_text(encoding="utf-8")
if "name:" not in text[:300]:
malformed.append(item.name)
return {
"ok": not missing and not malformed,
"canonical_dir": str(self.canonical_dir),
"count": len(names),
"missing": missing,
"malformed": malformed,
}
def sync_to(self, targets: list[Path], *, force: bool = True) -> int:
"""Copy canonical skills into adapter skill directories.
Returns the number of skill directories copied across all targets.
"""
copied = 0
for target in targets:
if target.resolve() == self.canonical_dir.resolve():
continue
target.mkdir(parents=True, exist_ok=True)
for item in self.list():
dst = target / item.name
if dst.exists() and force:
shutil.rmtree(dst)
if not dst.exists():
shutil.copytree(item.path.parent, dst)
copied += 1
return copied
def default_adapter_skill_dirs() -> list[Path]:
return [
REPO_ROOT / ".opencode" / "skills",
REPO_ROOT / ".agents" / "skills",
]
+65
View File
@@ -0,0 +1,65 @@
"""Source registry helpers for Phase 2 packets."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
def _source_key(source: dict[str, Any]) -> str:
return (source.get("url") or source.get("doi") or source.get("id") or "").strip()
def append_packet_sources(sources_path: Path, packet: dict[str, Any]) -> int:
"""Append packet sources to sources.jsonl, deduping by URL/DOI/id."""
sources_path.parent.mkdir(parents=True, exist_ok=True)
existing: set[str] = set()
if sources_path.exists():
for line in sources_path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
existing.add(_source_key(json.loads(line)))
except json.JSONDecodeError:
continue
written = 0
with sources_path.open("a", encoding="utf-8") as f:
for source in packet.get("sources") or []:
key = _source_key(source)
if not key or key in existing:
continue
existing.add(key)
f.write(json.dumps(source, ensure_ascii=False) + "\n")
written += 1
return written
def rebuild_sources_from_packets(project_root: Path) -> int:
"""Rebuild phase2/sources.jsonl from packet-level source metadata."""
packets_dir = project_root / "phase2" / "packets"
sources_path = project_root / "phase2" / "sources.jsonl"
sources_path.parent.mkdir(parents=True, exist_ok=True)
seen: set[str] = set()
rows: list[dict[str, Any]] = []
for packet_path in sorted(packets_dir.glob("*.json")):
try:
packet = json.loads(packet_path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
continue
for source in packet.get("sources") or []:
if not isinstance(source, dict):
continue
key = _source_key(source)
if not key or key in seen:
continue
seen.add(key)
rows.append(source)
sources_path.write_text(
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
encoding="utf-8",
)
return len(rows)
+229
View File
@@ -0,0 +1,229 @@
"""Task-card and evidence-packet primitives for v0.20 Phase 2."""
from __future__ import annotations
import json
import re
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
from scripts.runtime.methods import ResearchMethod
VALID_ROUTES = {"general", "scholar", "patents", "news"}
DEFAULT_AXES = ["literature", "regulatory", "patents", "market", "counter"]
AXIS_ROUTES = {
"literature": ["scholar", "general"],
"clinical": ["scholar", "general"],
"regulatory": ["general", "news"],
"patents": ["patents", "general"],
"market": ["news", "general"],
"china": ["news", "general"],
"counter": ["scholar", "general"],
"regulatory_gap": ["general", "news"],
"risk_classification": ["general", "scholar"],
"capa_design": ["general", "news"],
"ownership_timeline": ["general"],
"verification_evidence": ["general", "scholar"],
"process_flow": ["scholar", "general"],
"cqa_cpp": ["scholar", "general"],
"scale_up_risk": ["scholar", "general"],
"control_strategy": ["scholar", "general"],
"supply_chain": ["news", "general"],
"scientific_rationale": ["scholar", "general"],
"poc_evidence": ["scholar", "general"],
"ip_fto": ["patents", "general"],
"development_path": ["scholar", "general"],
"commercial_window": ["news", "general"],
"current_state": ["general"],
"capability_gap": ["general"],
"operating_model": ["general"],
"governance": ["general"],
"implementation_roadmap": ["general"],
}
@dataclass
class Chapter:
chapter_id: str
index: int
title: str
notes: str = ""
@dataclass
class TaskCard:
task_id: str
chapter_ids: list[str]
topic_axis: str
questions: list[str]
search_routes: list[str]
output_packet: str
preferred_model_role: str = "dr_analyst"
status: str = "pending"
dependencies: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def parse_framework_chapters(framework_text: str) -> list[Chapter]:
"""Extract Chinese or English chapter headings from a framework markdown."""
lines = framework_text.splitlines()
chapters: list[Chapter] = []
current: Chapter | None = None
note_lines: list[str] = []
heading_re = re.compile(
r"^#{1,3}\s*(?:第\s*)?(\d{1,2})\s*(?:章|[.)、:-])?\s*(.+?)\s*$",
re.IGNORECASE,
)
english_re = re.compile(r"^#{1,3}\s*chapter\s+(\d{1,2})[:.)\s-]+(.+?)\s*$", re.IGNORECASE)
for line in lines:
match = heading_re.match(line.strip()) or english_re.match(line.strip())
if match:
if current:
current.notes = "\n".join(note_lines).strip()
chapters.append(current)
index = int(match.group(1))
title = match.group(2).strip(" #")
current = Chapter(chapter_id=f"ch{index:02d}", index=index, title=title)
note_lines = []
elif current:
note_lines.append(line)
if current:
current.notes = "\n".join(note_lines).strip()
chapters.append(current)
return chapters
def _questions_for_axis(chapter: Chapter, axis: str) -> list[str]:
return [
f"围绕《{chapter.title}》从 {axis} 角度提炼可证伪的核心结论。",
"至少寻找两个 Tier 1-2 来源支撑主要结论;不足时标注待验证。",
"主动检索反方证据、限制条件或失败案例。",
]
def generate_task_cards(
slug: str,
framework_text: str,
*,
axes: list[str] | None = None,
method: ResearchMethod | None = None,
) -> list[TaskCard]:
del slug # slug is kept for call-site clarity and future namespacing.
chapters = parse_framework_chapters(framework_text)
selected_axes = axes or (method.task_axes if method else DEFAULT_AXES)
cards: list[TaskCard] = []
for chapter in chapters:
for axis in selected_axes:
routes = AXIS_ROUTES.get(axis, ["general"])
cards.append(
TaskCard(
task_id=f"{chapter.chapter_id}-{axis}",
chapter_ids=[chapter.chapter_id],
topic_axis=axis,
questions=_questions_for_axis(chapter, axis),
search_routes=routes,
output_packet=f"phase2/packets/{chapter.chapter_id}-{axis}.json",
preferred_model_role="dr_verifier" if axis == "counter" else "dr_analyst",
)
)
validate_task_cards(cards)
return cards
def detect_dependency_cycles(cards: list[TaskCard]) -> None:
graph = {card.task_id: card.dependencies for card in cards}
visiting: set[str] = set()
visited: set[str] = set()
def visit(node: str) -> None:
if node in visiting:
raise ValueError(f"dependency cycle detected at {node}")
if node in visited:
return
visiting.add(node)
for dep in graph.get(node, []):
visit(dep)
visiting.remove(node)
visited.add(node)
for task_id in graph:
visit(task_id)
def validate_task_cards(cards: list[TaskCard]) -> None:
seen: set[str] = set()
for card in cards:
if card.task_id in seen:
raise ValueError(f"duplicate task_id: {card.task_id}")
seen.add(card.task_id)
if not card.chapter_ids:
raise ValueError(f"{card.task_id}: chapter_ids required")
if not card.questions:
raise ValueError(f"{card.task_id}: questions required")
if not card.output_packet.endswith(".json"):
raise ValueError(f"{card.task_id}: output_packet must be json")
invalid_routes = sorted(set(card.search_routes) - VALID_ROUTES)
if invalid_routes:
raise ValueError(f"{card.task_id}: invalid search_routes {invalid_routes}")
missing_deps = sorted({dep for card in cards for dep in card.dependencies} - seen)
if missing_deps:
raise ValueError(f"unknown dependencies: {missing_deps}")
detect_dependency_cycles(cards)
def write_task_cards(path: Path, cards: list[TaskCard]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps([card.to_dict() for card in cards], ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
def load_task_cards(path: Path) -> list[TaskCard]:
data = json.loads(path.read_text(encoding="utf-8"))
cards = [TaskCard(**item) for item in data]
validate_task_cards(cards)
return cards
def validate_packet(packet: dict[str, Any]) -> None:
required = {
"task_id",
"claims",
"evidence_items",
"counter_evidence",
"source_ids",
"source_quality_notes",
"open_questions",
"raw_quotes_or_notes",
}
missing = sorted(required - set(packet))
if missing:
raise ValueError(f"packet missing fields: {missing}")
if not packet["claims"]:
raise ValueError("packet claims must not be empty")
if not packet["evidence_items"]:
raise ValueError("packet evidence_items must not be empty")
if not packet["counter_evidence"]:
raise ValueError("packet counter_evidence must not be empty")
declared = set(packet.get("source_ids") or [])
referenced: set[str] = set()
for section in ("claims", "counter_evidence"):
for item in packet.get(section) or []:
referenced.update(item.get("source_ids") or [])
for item in packet.get("evidence_items") or []:
if item.get("source_id"):
referenced.add(item["source_id"])
undeclared = sorted(referenced - declared)
if undeclared:
raise ValueError(f"packet source_ids referenced but not declared: {undeclared}")
packet_sources = packet.get("sources") or []
if packet_sources:
known_source_ids = {source.get("id") for source in packet_sources}
missing_sources = sorted(declared - known_source_ids)
if missing_sources:
raise ValueError(f"packet source_ids missing source metadata: {missing_sources}")
+261
View File
@@ -0,0 +1,261 @@
"""Python role workers for task-card execution."""
from __future__ import annotations
import json
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Callable, Protocol, Any
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
from scripts.runtime.skills import SkillRegistry
from scripts.runtime.tasks import TaskCard, validate_packet
from scripts.runtime.sources import append_packet_sources
class ChatClient(Protocol):
def chat_complete(self, **kwargs) -> str:
...
class SearchProvider(Protocol):
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
...
class ProjectSearchProvider:
"""Thin adapter over the project-owned search client."""
def __init__(self, *, strict_specialized: bool = True) -> None:
from scripts.lib.search_client import SearchClient
self.client = SearchClient(strict_specialized=strict_specialized)
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
if route == "scholar":
hits = self.client.scholar(query, num_results=num_results, year_low=2020)
elif route == "patents":
hits = self.client.patents(query, num_results=num_results)
elif route == "news":
hits = self.client.news(query, num_results=num_results, time_range="y")
else:
hits = self.client.search(query, num_results=num_results)
return [
{
"title": hit.title,
"url": hit.url,
"snippet": hit.snippet,
"route": route,
}
for hit in hits
]
def close(self) -> None:
self.client.close()
def _extract_json_object(text: str) -> dict:
stripped = text.strip()
if stripped.startswith("```"):
stripped = stripped.strip("`")
if stripped.startswith("json"):
stripped = stripped[4:].strip()
start = stripped.find("{")
end = stripped.rfind("}")
if start == -1 or end == -1 or end < start:
raise ValueError("worker response does not contain a JSON object")
return json.loads(stripped[start : end + 1])
def _safe_source_stem(task_id: str) -> str:
return re.sub(r"[^a-zA-Z0-9]+", "_", task_id).strip("_").lower()
def build_search_context(
card: TaskCard,
search_provider: SearchProvider,
*,
num_results_per_route: int = 5,
) -> dict[str, Any]:
candidate_sources: list[dict[str, Any]] = []
routes_used: list[str] = []
source_stem = _safe_source_stem(card.task_id)
idx = 1
query = " ".join(card.questions)
for route in card.search_routes:
routes_used.append(route)
hits = search_provider.search(query=query, route=route, num_results=num_results_per_route)
for hit in hits:
candidate_sources.append(
{
"id": f"src_{source_stem}_{idx:03d}",
"title": hit.get("title", ""),
"url": hit.get("url", ""),
"snippet": hit.get("snippet", ""),
"route": hit.get("route", route),
"tier": "Tier 2",
"score": 6,
}
)
idx += 1
return {"routes_used": routes_used, "candidate_sources": candidate_sources}
def build_packet_user_prompt(card: TaskCard, search_context: dict[str, Any] | None = None) -> str:
context = search_context or {"routes_used": [], "candidate_sources": []}
return (
"请根据以下 task card 产出一个证据包 JSON。\n"
"正式结论、summary、open_questions 用中文;英文原文摘录、source title、DOI/URL 可以保留英文。\n"
"必须主动包含 counter_evidence,且所有引用的 source_id 必须出现在 source_ids 中。\n\n"
"只能使用 candidate_sources 中的来源,不得编造 URL、DOI、trial ID 或 source_id。\n"
"输出 JSON 必须包含 sources 字段,且 sources 只能来自 candidate_sources。\n\n"
f"{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
"只输出 JSON,不要输出 Markdown 解释。"
)
def build_packet_repair_prompt(
*,
card: TaskCard,
raw_response: str,
error: Exception,
search_context: dict[str, Any] | None = None,
) -> str:
context = search_context or {"routes_used": [], "candidate_sources": []}
return (
"请修复上一次 evidence packet 输出,使其成为合法且通过 schema 校验的 JSON。\n"
"只输出 JSON 对象,不要输出 Markdown、解释或代码块。\n"
"保留中文主写作;英文只允许出现在来源标题、URL、DOI、原文摘录或检索笔记中。\n"
"不得编造 candidate_sources 以外的来源、URL、DOI、trial ID 或 source_id。\n\n"
f"Schema error:\n{error}\n\n"
f"Task card:\n{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
f"Previous raw response:\n{raw_response[:12000]}"
)
class PacketWorker:
def __init__(
self,
*,
role: RoleDefinition,
client: ChatClient,
search_provider: SearchProvider | None = None,
skill_registry: SkillRegistry | None = None,
num_results_per_route: int = 5,
) -> None:
self.role = role
self.client = client
self.search_provider = search_provider
self.skill_registry = skill_registry or SkillRegistry()
self.num_results_per_route = num_results_per_route
def _system_prompt(self) -> str:
skill_texts = []
for name in self.role.skills:
try:
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
except FileNotFoundError:
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
return (
"你是 Deep Research v0.20 Python runtime 的证据包 worker。\n"
"你的唯一任务是把一个 task card 转换为结构化 evidence packet。\n"
"遵循中文主写作原则;不要写章节正文;不要编造 URL、DOI、trial ID 或 source_id。\n\n"
"搜索只能走项目 Python search gateway 或调用方提供的 search_context;不要直接使用 Tavily MCP、browser MCP、平台 web search 或任何需要用户权限确认的外部搜索工具。\n\n"
+ "\n\n".join(skill_texts)
)
def run(self, card: TaskCard) -> dict:
search_context = None
if self.search_provider:
search_context = build_search_context(
card,
self.search_provider,
num_results_per_route=self.num_results_per_route,
)
raw = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_packet_user_prompt(card, search_context),
temperature=self.role.temperature,
max_tokens=self.role.max_tokens,
tag=f"packet:{card.task_id}",
)
try:
packet = _extract_json_object(raw)
validate_packet(packet)
return packet
except Exception as error:
repaired = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_packet_repair_prompt(
card=card,
raw_response=raw,
error=error,
search_context=search_context,
),
temperature=0,
max_tokens=self.role.max_tokens,
tag=f"packet-repair:{card.task_id}",
)
packet = _extract_json_object(repaired)
validate_packet(packet)
return packet
def _write_packet_error(project_root: Path, card: TaskCard, error: Exception) -> None:
path = project_root / "phase2" / "packet_errors" / f"{card.task_id}.json"
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"task_id": card.task_id,
"status": "failed",
"error": str(error),
"output_packet": card.output_packet,
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def run_packet_workers(
*,
project_root: Path,
cards: list[TaskCard],
runtime: RuntimeProfile,
client_factory: Callable[[RoleDefinition], ChatClient],
search_provider_factory: Callable[[], SearchProvider] | None = None,
workers: int,
) -> int:
role = runtime.role_for_task("evidence_packet")
max_workers = max(1, min(workers, role.max_concurrency))
def run_one(card: TaskCard) -> tuple[TaskCard, dict | None, Exception | None]:
search_provider = search_provider_factory() if search_provider_factory else None
try:
worker = PacketWorker(role=role, client=client_factory(role), search_provider=search_provider)
return card, worker.run(card), None
except Exception as error:
return card, None, error
finally:
close = getattr(search_provider, "close", None)
if close:
close()
written = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = [pool.submit(run_one, card) for card in cards]
for future in as_completed(futures):
card, packet, error = future.result()
if error is not None:
_write_packet_error(project_root, card, error)
continue
if packet is None:
_write_packet_error(project_root, card, RuntimeError("packet worker returned no packet"))
continue
path = project_root / card.output_packet
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
append_packet_sources(project_root / "phase2" / "sources.jsonl", packet)
written += 1
return written
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""v0.20 Python-core regression checks."""
from __future__ import annotations
import json
import subprocess
import sys
import tempfile
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
def run(cmd: list[str]) -> None:
print("$ " + " ".join(cmd))
result = subprocess.run(cmd, cwd=REPO_ROOT, check=False, text=True, capture_output=True)
if result.stdout:
print(result.stdout.rstrip())
if result.stderr:
print(result.stderr.rstrip(), file=sys.stderr)
if result.returncode != 0:
raise SystemExit(result.returncode)
def make_fixture(root: Path) -> Path:
project = root / "v020-fixture"
(project / "phase4").mkdir(parents=True, exist_ok=True)
(project / "phase4" / "final_zh.md").write_text(
"# v0.20 回归测试报告\n\n正文引用占位。\n",
encoding="utf-8",
)
return project
def main() -> int:
python = sys.executable
run([python, "scripts/dr.py", "skills", "validate"])
run([python, "scripts/dr.py", "models", "--profile", "medium", "--json"])
with tempfile.TemporaryDirectory(prefix="deep-research-v020-") as tmp:
tmp_root = Path(tmp)
run(
[
python,
"scripts/dr.py",
"init",
"v0.20 fixture",
"--slug",
"v020-fixture",
"--projects-dir",
str(tmp_root),
"--method",
"mckinsey_market",
]
)
project = make_fixture(tmp_root)
run([python, "scripts/dr.py", "frame", str(project)])
run([python, "scripts/dr.py", "approve", str(project)])
run([python, "scripts/dr.py", "research", str(project), "--workers", "2", "--dry-run"])
run([python, "scripts/dr.py", "research", str(project), "--workers", "2"])
run([python, "scripts/dr.py", "review", str(project)])
run([python, "scripts/dr.py", "finalize", str(project), "--dry-run"])
print("v0.20 regression PASS")
return 0
if __name__ == "__main__":
raise SystemExit(main())