v0.20 alpha skill-driven python core

This commit is contained in:
kai
2026-05-06 16:26:41 +08:00
parent d1169646b8
commit db626f1d58
87 changed files with 5213 additions and 2865 deletions
+6
View File
@@ -0,0 +1,6 @@
"""v0.20 Python runtime core for Deep Research.
The runtime layer is intentionally platform-neutral: OpenCode, Codex, and
Claude Code should call into these modules instead of owning orchestration.
"""
+46
View File
@@ -0,0 +1,46 @@
"""Project artifact helpers shared by the Python runtime."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[2]
PROJECTS_DIR = REPO_ROOT / "projects"
def resolve_project(project: str | Path) -> Path:
p = Path(project)
if p.is_dir():
return p.resolve()
candidate = PROJECTS_DIR / str(project)
if candidate.is_dir():
return candidate.resolve()
raise FileNotFoundError(f"project not found: {project}")
def load_manifest(project_root: Path) -> dict[str, Any]:
path = project_root / "manifest.json"
if not path.exists():
raise FileNotFoundError(f"manifest not found: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def write_manifest(project_root: Path, manifest: dict[str, Any]) -> None:
path = project_root / "manifest.json"
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def ensure_phase_dirs(project_root: Path) -> None:
for rel in (
"phase1",
"phase2/drafts",
"phase2/evidence",
"phase2/packets",
"phase3",
"phase4",
):
(project_root / rel).mkdir(parents=True, exist_ok=True)
+202
View File
@@ -0,0 +1,202 @@
"""Chapter brief aggregation and Chinese chapter assembly."""
from __future__ import annotations
import json
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Callable
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
from scripts.runtime.skills import SkillRegistry
from scripts.runtime.tasks import load_task_cards, validate_packet
from scripts.runtime.workers import ChatClient
def validate_chapter_brief(brief: dict) -> None:
required = {
"chapter_id",
"chapter_title",
"packet_ids",
"core_claims",
"evidence_items",
"counter_evidence",
"source_ids",
"open_questions",
"assembly_notes",
}
missing = sorted(required - set(brief))
if missing:
raise ValueError(f"chapter brief missing fields: {missing}")
if not brief["chapter_id"]:
raise ValueError("chapter_id required")
if not brief["packet_ids"]:
raise ValueError("chapter brief requires at least one packet")
if not brief["core_claims"]:
raise ValueError("chapter brief requires core_claims")
if not brief["evidence_items"]:
raise ValueError("chapter brief requires evidence_items")
if not brief["counter_evidence"]:
raise ValueError("chapter brief requires counter_evidence")
def validate_chapter_markdown_citations(markdown: str, brief: dict) -> None:
validate_chapter_brief(brief)
cited = set(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", markdown))
allowed = set(brief.get("source_ids") or [])
unknown = sorted(cited - allowed)
if unknown:
raise ValueError(f"unknown citation ids in {brief['chapter_id']}: {unknown}")
def _chapter_title_from_id(chapter_id: str) -> str:
try:
index = int(chapter_id.replace("ch", ""))
return f"{index}"
except ValueError:
return chapter_id
def build_chapter_briefs(project_root: Path) -> list[dict]:
cards = load_task_cards(project_root / "phase2" / "task_cards.json")
grouped: dict[str, list[tuple[str, dict]]] = {}
for card in cards:
packet_path = project_root / card.output_packet
if not packet_path.exists():
continue
packet = json.loads(packet_path.read_text(encoding="utf-8"))
validate_packet(packet)
for chapter_id in card.chapter_ids:
grouped.setdefault(chapter_id, []).append((card.task_id, packet))
briefs: list[dict] = []
out_dir = project_root / "phase2" / "chapter_briefs"
out_dir.mkdir(parents=True, exist_ok=True)
for chapter_id in sorted(grouped):
packet_pairs = sorted(grouped[chapter_id], key=lambda item: item[0])
packet_ids = [item[0] for item in packet_pairs]
packets = [item[1] for item in packet_pairs]
source_ids = sorted({sid for packet in packets for sid in packet.get("source_ids", [])})
brief = {
"chapter_id": chapter_id,
"chapter_title": _chapter_title_from_id(chapter_id),
"packet_ids": packet_ids,
"core_claims": [claim for packet in packets for claim in packet.get("claims", [])],
"evidence_items": [item for packet in packets for item in packet.get("evidence_items", [])],
"counter_evidence": [item for packet in packets for item in packet.get("counter_evidence", [])],
"source_ids": source_ids,
"open_questions": [q for packet in packets for q in packet.get("open_questions", [])],
"assembly_notes": [
"用中文写正式章节,英文仅保留在必要的来源标题、原文摘录、DOI/URL 中。",
"避免碎片化:不要按 packet 逐段堆砌,要先提炼本章主线,再组织证据。",
"每个事实、数字和关键判断都必须保留 [src_xxx] 引用。",
"必须纳入 counter_evidence,并说明它如何影响结论置信度。",
],
}
validate_chapter_brief(brief)
(out_dir / f"{chapter_id}.json").write_text(
json.dumps(brief, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
briefs.append(brief)
return briefs
def build_chapter_user_prompt(brief: dict) -> str:
return (
"请根据以下 chapter brief 写一章正式中文 Markdown 正文。\n"
"目标是形成一个完整章节,而不是 packet 摘要。避免碎片化,按金字塔结构组织:章首先给结论,再用证据支撑。\n"
"要求:标题必须是观点型判断;每个数字和事实保留 [src_xxx];纳入反方证据;不要出现调度元数据。\n"
"禁止写空泛咨询腔。每个二级小节都必须至少落下 2 个具体审计发现、法规要求、SOP/记录/参数/现场观察或整改证据;不要只写原则。\n"
"正文末尾必须增加“证据落点与待补证据”小节,用表格列出:关键判断、已使用证据 source_id、已落地整改动作、仍缺证据。若证据不足,直接标注需回炉 Phase 2,不要用泛泛表述补齐。\n"
"只输出 Markdown,不要输出解释。\n\n"
f"{json.dumps(brief, ensure_ascii=False, indent=2)}"
)
class ChapterAssemblyWorker:
def __init__(
self,
*,
role: RoleDefinition,
client: ChatClient,
skill_registry: SkillRegistry | None = None,
) -> None:
self.role = role
self.client = client
self.skill_registry = skill_registry or SkillRegistry()
def _system_prompt(self) -> str:
skill_texts = []
for name in self.role.skills:
try:
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
except FileNotFoundError:
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
return (
"你是 Deep Research v0.20 的中文章节组装 worker。\n"
"你的职责是把结构化证据包收束成连贯章节,解决并发研究造成的碎片化。\n"
"不得编造来源,不得删除关键反方证据。\n\n"
+ "\n\n".join(skill_texts)
)
def write_chapter(self, *, project_root: Path, brief: dict) -> Path:
validate_chapter_brief(brief)
markdown = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_chapter_user_prompt(brief),
temperature=self.role.temperature,
max_tokens=self.role.max_tokens,
tag=f"chapter:{brief['chapter_id']}",
)
validate_chapter_markdown_citations(markdown, brief)
out = project_root / "phase2" / "drafts" / f"{brief['chapter_id']}.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(markdown.rstrip() + "\n", encoding="utf-8")
return out
def _write_chapter_error(project_root: Path, brief: dict, error: Exception) -> None:
path = project_root / "phase2" / "chapter_errors" / f"{brief.get('chapter_id', 'unknown')}.json"
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"chapter_id": brief.get("chapter_id"),
"status": "failed",
"error": str(error),
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def run_chapter_assembly_workers(
*,
project_root: Path,
briefs: list[dict],
runtime: RuntimeProfile,
client_factory: Callable[[RoleDefinition], ChatClient],
workers: int,
) -> int:
role = runtime.role_for_task("chapter_assembly")
max_workers = max(1, min(workers, role.max_concurrency))
def run_one(brief: dict) -> tuple[dict, Path | None, Exception | None]:
try:
worker = ChapterAssemblyWorker(role=role, client=client_factory(role))
return brief, worker.write_chapter(project_root=project_root, brief=brief), None
except Exception as error:
return brief, None, error
written = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = [pool.submit(run_one, brief) for brief in briefs]
for future in as_completed(futures):
brief, path, error = future.result()
if error is not None:
_write_chapter_error(project_root, brief, error)
continue
if path is None:
_write_chapter_error(project_root, brief, RuntimeError("chapter worker returned no output path"))
continue
written += 1
return written
+229
View File
@@ -0,0 +1,229 @@
"""Phase 0 user-provided material ingestion."""
from __future__ import annotations
import base64
import os
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import requests
DEFAULT_FIRERED_OCR_ENDPOINT = "http://192.168.50.100:8001"
DEFAULT_OCR_MAX_PAGES = 50
@dataclass(frozen=True)
class OcrResult:
text: str
pages_processed: int
output_path: Path
def safe_filename(path: Path) -> str:
name = path.name.strip()
return name or "material"
def extract_pdf_text(path: Path) -> tuple[str, int, bool]:
from pypdf import PdfReader
reader = PdfReader(str(path))
chunks: list[str] = []
for index, page in enumerate(reader.pages, start=1):
text = (page.extract_text() or "").strip()
if text:
chunks.append(f"\n\n## Page {index}\n\n{text}")
combined = "".join(chunks).strip()
ocr_required = len(combined) < max(20, len(reader.pages) * 20)
return combined, len(reader.pages), ocr_required
def ocr_endpoint_from_env() -> str:
return os.environ.get("DEEP_RESEARCH_OCR_ENDPOINT", DEFAULT_FIRERED_OCR_ENDPOINT).rstrip("/")
def ocr_max_pages_from_env() -> int:
raw = os.environ.get("DEEP_RESEARCH_OCR_MAX_PAGES")
if not raw:
return DEFAULT_OCR_MAX_PAGES
try:
return max(1, int(raw))
except ValueError:
return DEFAULT_OCR_MAX_PAGES
def render_pdf_pages(pdf_path: Path, output_dir: Path, *, max_pages: int) -> list[Path]:
import fitz
pages_dir = output_dir / f"{pdf_path.stem}.ocr-pages"
pages_dir.mkdir(parents=True, exist_ok=True)
image_paths: list[Path] = []
doc = fitz.open(pdf_path)
try:
for index, page in enumerate(doc[:max_pages], start=1):
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
image_path = pages_dir / f"page-{index:03d}.png"
pix.save(image_path)
image_paths.append(image_path)
finally:
doc.close()
return image_paths
def data_url_for_image(path: Path) -> str:
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
return f"data:image/png;base64,{encoded}"
def call_firered_ocr(image_path: Path, *, endpoint: str) -> str:
payload = {
"model": "firered-ocr",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "请识别图片中的全部文字,保持原有顺序,只输出文字。"},
{"type": "image_url", "image_url": {"url": data_url_for_image(image_path)}},
],
}
],
"temperature": 0,
"max_tokens": 3000,
}
response = requests.post(f"{endpoint.rstrip('/')}/v1/chat/completions", json=payload, timeout=60)
if not response.ok:
raise RuntimeError(f"{response.status_code} {response.text[:500]}")
data = response.json()
return str(data["choices"][0]["message"].get("content") or "").strip()
def ocr_pdf_with_firered(*, pdf_path: Path, output_dir: Path, endpoint: str, max_pages: int) -> OcrResult:
image_paths = render_pdf_pages(pdf_path, output_dir, max_pages=max_pages)
chunks: list[str] = []
for index, image_path in enumerate(image_paths, start=1):
text = call_firered_ocr(image_path, endpoint=endpoint)
if text:
chunks.append(f"\n\n## OCR Page {index}\n\n{text}")
combined = "".join(chunks).strip()
output_path = output_dir / f"{pdf_path.stem}.ocr.md"
body = [
f"# OCR Material: {pdf_path.name}",
"",
f"- source_path: {pdf_path}",
f"- endpoint: {endpoint}",
f"- pages_processed: {len(image_paths)}",
"",
combined or "OCR 未返回可用文本。",
"",
]
output_path.write_text("\n".join(body), encoding="utf-8")
return OcrResult(text=combined, pages_processed=len(image_paths), output_path=output_path)
def ingest_input_materials(project_root: Path, materials: list[str] | None) -> list[dict[str, Any]]:
inventory: list[dict[str, Any]] = []
if not materials:
return inventory
inputs_dir = project_root / "phase0" / "inputs"
extracted_dir = project_root / "phase0" / "extracted"
ocr_endpoint = ocr_endpoint_from_env()
ocr_max_pages = ocr_max_pages_from_env()
inputs_dir.mkdir(parents=True, exist_ok=True)
extracted_dir.mkdir(parents=True, exist_ok=True)
for raw in materials:
source = Path(raw).expanduser()
if not source.exists():
inventory.append({"kind": "note", "note": raw})
continue
copied = inputs_dir / safe_filename(source)
shutil.copy2(source, copied)
item: dict[str, Any] = {
"kind": source.suffix.lower().lstrip(".") or "file",
"source_path": str(source),
"copied_to": str(copied.relative_to(project_root)),
"size_bytes": source.stat().st_size,
}
if source.suffix.lower() == ".pdf":
text, pages, ocr_required = extract_pdf_text(source)
extracted = extracted_dir / f"{source.stem}.md"
ocr_result: OcrResult | None = None
ocr_error: str | None = None
if ocr_required:
try:
ocr_result = ocr_pdf_with_firered(
pdf_path=source,
output_dir=extracted_dir,
endpoint=ocr_endpoint,
max_pages=min(pages, ocr_max_pages),
)
if ocr_result.text:
text = "\n\n".join(part for part in [text, ocr_result.text] if part)
except Exception as exc: # noqa: BLE001 - ingestion should not block project init.
ocr_error = str(exc)
body = [
f"# Extracted Material: {source.name}",
"",
f"- source_path: {source}",
f"- copied_to: {copied.relative_to(project_root)}",
f"- pages: {pages}",
f"- ocr_required: {str(ocr_required).lower()}",
f"- ocr_status: {'completed' if ocr_result else 'failed' if ocr_error else 'not_required'}",
"",
text or "未能从 PDF 直接抽取文本;该材料可能需要 OCR。",
"",
]
if ocr_error:
body.extend(["## OCR Error", "", ocr_error, ""])
extracted.write_text("\n".join(body), encoding="utf-8")
item.update(
{
"pages": pages,
"extracted_to": str(extracted.relative_to(project_root)),
"text_chars": len(text),
"ocr_required": ocr_required,
"ocr_status": "completed" if ocr_result else "failed" if ocr_error else "not_required",
}
)
if ocr_result:
item.update(
{
"ocr_endpoint": ocr_endpoint,
"ocr_pages_processed": ocr_result.pages_processed,
"ocr_extracted_to": str(ocr_result.output_path.relative_to(project_root)),
"ocr_text_chars": len(ocr_result.text),
}
)
if ocr_error:
item["ocr_error"] = ocr_error
else:
item["ocr_required"] = source.suffix.lower() in {".png", ".jpg", ".jpeg", ".tif", ".tiff"}
inventory.append(item)
return inventory
def render_material_inventory(inventory: list[dict[str, Any]]) -> str:
if not inventory:
return "- 暂无;可通过 `--input-material` 加入审计报告、问题清单或内部记录。"
lines: list[str] = []
for item in inventory:
if item.get("kind") == "note":
lines.append(f"- 备注:{item.get('note', '')}")
continue
marker = ";需要 OCR" if item.get("ocr_required") else ""
ocr = f"OCR{item.get('ocr_status')}" if item.get("ocr_status") else ""
extracted = item.get("extracted_to")
extra = f";抽取文本:{extracted}" if extracted else ""
lines.append(
f"- {item.get('copied_to')}{item.get('kind')}{item.get('size_bytes', 0)} bytes{extra}{marker}{ocr}"
)
return "\n".join(lines)
+60
View File
@@ -0,0 +1,60 @@
"""Research method registry for Phase 1 framework selection."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_METHOD_CONFIG = REPO_ROOT / "configs" / "research_methods.yaml"
@dataclass(frozen=True)
class ResearchMethod:
key: str
name: str
best_for: list[str]
structure_principle: str
task_axes: list[str]
framework_sections: list[str]
class ResearchMethodRegistry:
def __init__(self, path: Path | None = None) -> None:
self.path = path or DEFAULT_METHOD_CONFIG
self._data = self._load()
def _load(self) -> dict[str, Any]:
if not self.path.exists():
raise FileNotFoundError(f"research method config not found: {self.path}")
data = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {}
if not isinstance(data, dict) or "methods" not in data:
raise ValueError(f"invalid research method config: {self.path}")
return data
@property
def default_method(self) -> str:
return (self._data.get("defaults") or {}).get("method", "mckinsey_market")
def list_names(self) -> list[str]:
return sorted((self._data.get("methods") or {}).keys())
def get(self, key: str | None = None) -> ResearchMethod:
selected = key or self.default_method
methods = self._data.get("methods") or {}
if selected not in methods:
raise KeyError(f"unknown research_method: {selected}")
item = methods[selected] or {}
return ResearchMethod(
key=selected,
name=item.get("name", selected),
best_for=list(item.get("best_for") or []),
structure_principle=item.get("structure_principle", ""),
task_axes=list(item.get("task_axes") or []),
framework_sections=list(item.get("framework_sections") or []),
)
+79
View File
@@ -0,0 +1,79 @@
"""Deterministic orchestration helpers for v0.20."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from scripts.runtime.artifacts import ensure_phase_dirs, load_manifest, write_manifest
from scripts.runtime.methods import ResearchMethodRegistry
from scripts.runtime.tasks import generate_task_cards, write_task_cards
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def create_phase2_task_cards(
project_root: Path,
*,
axes: list[str] | None = None,
dry_run: bool = False,
) -> list[dict[str, object]]:
framework = project_root / "phase1" / "framework.md"
if not framework.exists():
raise FileNotFoundError(f"framework not found: {framework}")
method_key = None
if (project_root / "manifest.json").exists():
method_key = load_manifest(project_root).get("research_method")
method = ResearchMethodRegistry().get(method_key)
cards = generate_task_cards(project_root.name, framework.read_text(encoding="utf-8"), axes=axes, method=method)
if not dry_run:
ensure_phase_dirs(project_root)
write_task_cards(project_root / "phase2" / "task_cards.json", cards)
manifest = load_manifest(project_root)
phase2 = manifest.setdefault("phase2", {})
phase2.update(
{
"status": "in_progress",
"runtime": "python-core-v0.20",
"research_method": method.key,
"task_cards_path": "phase2/task_cards.json",
"task_cards_total": len(cards),
"updated_at": utc_now_iso(),
}
)
write_manifest(project_root, manifest)
return [card.to_dict() for card in cards]
def write_placeholder_packets(
project_root: Path,
task_cards: list[dict[str, object]],
*,
dry_run: bool = False,
) -> int:
"""Create packet skeletons for manual/API completion.
This keeps the first v0.20 implementation deterministic and resumable; LLM
calls can later fill the same schema without changing downstream readers.
"""
count = 0
for card in task_cards:
packet_path = project_root / str(card["output_packet"])
packet = {
"task_id": card["task_id"],
"claims": [],
"evidence_items": [],
"counter_evidence": [],
"source_ids": [],
"source_quality_notes": [],
"open_questions": ["待由 Python role worker 调用模型补全。"],
"raw_quotes_or_notes": [],
}
if not dry_run:
packet_path.parent.mkdir(parents=True, exist_ok=True)
packet_path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
count += 1
return count
+341
View File
@@ -0,0 +1,341 @@
"""Phase 1 project initialization and framework generation."""
from __future__ import annotations
import hashlib
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from scripts.runtime.artifacts import PROJECTS_DIR, ensure_phase_dirs, load_manifest, write_manifest
from scripts.runtime.materials import ingest_input_materials, render_material_inventory
from scripts.runtime.methods import ResearchMethod, ResearchMethodRegistry
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def slugify_topic(topic: str) -> str:
slug = re.sub(r"[^a-zA-Z0-9]+", "-", topic.lower()).strip("-")
if slug:
return slug[:80]
digest = hashlib.sha1(topic.encode("utf-8")).hexdigest()[:8]
return f"research-{digest}"
def create_project(
*,
topic: str,
slug: str | None = None,
projects_dir: Path = PROJECTS_DIR,
method_key: str | None = None,
report_type: str = "research",
model_profile: str = "medium",
target_words: int = 30000,
input_materials: list[str] | None = None,
) -> Path:
method = ResearchMethodRegistry().get(method_key)
project_slug = slug or slugify_topic(topic)
project_root = projects_dir / project_slug
if project_root.exists():
raise FileExistsError(f"project already exists: {project_root}")
ensure_phase_dirs(project_root)
(project_root / "phase0" / "inputs").mkdir(parents=True, exist_ok=True)
(project_root / "phase0" / "extracted").mkdir(parents=True, exist_ok=True)
material_inventory = ingest_input_materials(project_root, input_materials)
now = utc_now_iso()
manifest: dict[str, Any] = {
"version": "0.20.0",
"runtime": "python-core-v0.20",
"topic": topic,
"slug": project_slug,
"report_title": topic,
"type": report_type,
"work_language": "zh",
"model_profile": model_profile,
"research_method": method.key,
"target_words": target_words,
"input_materials": input_materials or [],
"material_inventory": material_inventory,
"created_at": now,
"updated_at": now,
"phase1": {
"status": "initialized",
"approved": False,
"requires_user_interview": True,
"material_brief_path": "phase1/material_brief.md",
},
"phase2": {"status": "pending"},
"phase3": {"status": "pending"},
"phase4": {"status": "pending"},
}
write_manifest(project_root, manifest)
_write_interview_seed(project_root, manifest, method)
write_material_brief(project_root, manifest, method)
return project_root
def _write_interview_seed(project_root: Path, manifest: dict[str, Any], method: ResearchMethod) -> None:
material_text = render_material_inventory(manifest.get("material_inventory") or [])
text = (
f"# Phase 1 访谈记录\n\n"
f"- 主题:{manifest['topic']}\n"
f"- 研究方法:{method.key} - {method.name}\n"
f"- 报告类型:{manifest['type']}\n"
f"- 目标字数:{manifest['target_words']}\n"
f"- 工作语言:中文主写作;检索关键词、证据摘录和来源笔记可保留英文。\n\n"
f"## 已提供材料\n\n{material_text}\n\n"
"## 后续访谈问题\n\n"
"1. 本报告最重要的决策用途是什么?\n"
"2. 是否有必须覆盖或必须排除的公司、产品、工艺、市场或法规范围?\n"
"3. 结论偏好是战略建议、风险清单、投资判断,还是执行路线图?\n"
)
(project_root / "phase1" / "interview.md").write_text(text, encoding="utf-8")
def _material_excerpt(project_root: Path, rel_path: str, *, max_chars: int = 1200) -> str:
path = project_root / rel_path
if not path.exists():
return "(未找到抽取文本)"
text = path.read_text(encoding="utf-8")
compact = "\n".join(line.rstrip() for line in text.splitlines() if line.strip())
return compact[:max_chars] + ("..." if len(compact) > max_chars else "")
def _derive_material_observations(project_root: Path, inventory: list[dict[str, Any]]) -> list[str]:
combined_parts: list[str] = []
for item in inventory:
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
if rel and (project_root / rel).exists():
combined_parts.append((project_root / rel).read_text(encoding="utf-8"))
text = "\n".join(combined_parts)
checks = [
("审计范围覆盖生产管理、原液、制剂和无菌相关模块,报告需要同时处理 GMP 合规、工艺转移和运营协同,而不是只写质量体系。", ["生产管理", "原液", "制剂", "无菌"]),
("材料显示高风险项为 0、中风险项为 1,适合采用“商业化 readiness 与系统成熟度差距”而非“体系失控”作为初始假设。", ["高风险 0", "中风险1", "低风险7"]),
("商业化经验、无菌保障细节、文件要求与执行一致性是需要访谈确认的主线风险。", ["商业化经验不足", "无菌保障", "文件要求与执行一致性"]),
("工艺规程、批记录、CPP/CQA、VMPR/VMP、验证主计划等内容反复出现,说明工艺验证和商业化文件体系可能是 Phase 2 的重点证据轴。", ["CPP", "CQA", "VMPR", "VMP"]),
("温度、压差、WFI、冷却段微生物、RABS/ORABS、first air、APS 等无菌和设施细节需要映射到 EU Annex 1、NMPA GMP 和企业 SOP。", ["温度", "压差", "WFI", "APS"]),
("复盘材料包含责任人和局部答复,后续整改路线图应尽量回填 owner、期限、关闭证据和复核机制。", ["填写人", "是否已经回答完整", "整改"]),
]
observations = [message for message, needles in checks if any(needle in text for needle in needles)]
return observations or ["材料已导入但尚未形成足够结构化判断;需要先访谈确认研究用途、范围和优先级。"]
def write_material_brief(
project_root: Path,
manifest: dict[str, Any] | None = None,
method: ResearchMethod | None = None,
) -> Path:
"""Write a Phase 0/1 material brief that must be reviewed before Phase 2."""
manifest = manifest or load_manifest(project_root)
method = method or ResearchMethodRegistry().get(manifest.get("research_method"))
inventory = manifest.get("material_inventory") or []
lines = [
f"# Phase 0 材料简报:{manifest.get('topic', project_root.name)}",
"",
"status: 待用户确认",
f"research_method: {method.key}",
"",
"## 已导入材料",
"",
render_material_inventory(inventory),
"",
"## 材料初步解读",
"",
"以下内容由 Python core 从已落盘材料抽样生成,只作为访谈起点;不得直接视为最终结论。",
"",
]
lines.extend(["## 初步问题聚类(待访谈确认)", ""])
for observation in _derive_material_observations(project_root, inventory):
lines.append(f"- {observation}")
lines.append("")
lines.append("## 材料摘录")
lines.append("")
for item in inventory:
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
if not rel:
continue
lines.extend(
[
f"### {Path(rel).name}",
"",
_material_excerpt(project_root, rel),
"",
]
)
lines.extend(
[
"## 建议访谈确认点",
"",
"1. 本报告的最重要用途是什么:内部整改、客户沟通、董事会决策,还是外部审计准备?",
"2. 哪些审计发现最需要优先展开:无菌保障、工艺验证、数据完整性、质量体系闭环,还是运营协同?",
"3. 是否存在必须排除或脱敏的项目、人员、客户、产品或工艺信息?",
"4. 短中长期整改的时间边界如何定义,例如 30/90/180 天,还是按临床/商业化里程碑划分?",
"5. 是否需要把 NMPA、FDA、EMA、ICH、WHO 的法规基线分别映射到整改责任人和证据包?",
"",
"## Gate",
"",
"请用户确认本材料简报与访谈问题后,再生成或批准 `phase1/framework.md` 并进入 Phase 2。",
"",
]
)
out = project_root / "phase1" / "material_brief.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
return out
CHAPTER_TEMPLATES: dict[str, list[str]] = {
"mckinsey_market": [
"核心结论先行界定市场机会与约束",
"临床与真实世界证据决定需求天花板",
"监管路径和支付环境重塑商业化节奏",
"竞争格局正在从单点产品转向组合能力",
"专利与技术壁垒决定长期利润池",
"中国市场的准入和供给能力形成独立变量",
"资本市场预期与基本面之间存在可验证偏差",
"反方证据限定结论边界并提示回撤风险",
"战略选择应围绕资源约束排序",
"执行路线图需要把证据缺口转化为行动清单",
],
"gmp_gap_assessment": [
"监管基线决定整改范围而非企业主观偏好",
"现状差距需要按法规条款和业务流程双重定位",
"质量风险分级决定 CAPA 优先级",
"根因分析质量决定整改能否闭环",
"CAPA 设计必须绑定责任人、证据和期限",
"验证计划决定整改是否可被审计接受",
"供应商和外包管理常是系统性缺口放大器",
"数据完整性风险需要独立成章处理",
"实施路线图需要平衡停线风险与合规风险",
"管理层治理机制决定整改能否持续",
],
"cmc_process_risk": [
"工艺流程图是识别放大风险的起点",
"CQA 与 CPP 的映射决定控制策略质量",
"放大过程的失效模式集中在传质、混合和稳定性",
"分析方法和放行标准决定证据可信度",
"技术转移风险来自知识隐性化和现场差异",
"供应链约束会改变工艺控制边界",
"偏差和变更管理决定商业化后的韧性",
"监管沟通策略需要提前固化关键假设",
"反方证据限定平台工艺可复制性",
"CMC 路线图需要把风险转化为验证实验",
],
"rd_go_no_go": [
"科学假设强度决定项目是否值得进入下一阶段",
"POC 证据需要同时证明有效性和可转化性",
"安全性窗口决定适应症与人群选择",
"IP 与 FTO 风险决定商业化自由度",
"开发路径需要把关键不确定性前置验证",
"竞争窗口决定速度是否仍有战略价值",
"CMC 与临床运营能力影响真实可行性",
"反方证据决定 go/no-go 阈值",
"投资强度应与证据成熟度匹配",
"决策门槛需要形成可执行检查表",
],
"management_consulting": [
"现状诊断需要区分症状、根因和约束条件",
"能力差距决定组织改进优先级",
"流程断点揭示跨部门协作成本",
"治理结构决定决策速度和责任清晰度",
"运营模型需要匹配战略目标而非照搬标杆",
"数字化工具只有嵌入流程才产生价值",
"绩效指标需要避免局部最优",
"变革阻力本身是方案设计输入",
"路线图需要把 quick wins 与系统建设分层",
"落地机制决定咨询建议能否转化为成果",
],
"gmp_quality_operations_diagnosis": [
"现场审计发现需要先转化为可验证的系统性问题图谱",
"法规基线决定质量体系差距的严重度与整改边界",
"生产工艺体系风险来自流程、设施、公用系统和验证证据的耦合缺口",
"偏差、变更、CAPA 和数据完整性决定质量系统能否闭环",
"人员能力与质量文化决定制度是否真正落地",
"运营管理问题需要区分组织、流程、会议机制和指标体系缺口",
"跨部门协同断点会放大 GMP 风险和交付风险",
"标杆实践应转化为短中长期整改组合而非口号",
"整改路线图必须绑定责任、优先级、证据和复核机制",
"管理层治理机制决定白帆能否从一次整改转向持续改进",
],
}
def render_framework(project_root: Path, *, method_key: str | None = None, chapter_count: int = 10) -> Path:
manifest = load_manifest(project_root)
registry = ResearchMethodRegistry()
method = registry.get(method_key or manifest.get("research_method"))
if method_key:
manifest["research_method"] = method.key
titles = CHAPTER_TEMPLATES.get(method.key) or CHAPTER_TEMPLATES["mckinsey_market"]
chapter_count = max(8, min(15, chapter_count))
selected = titles[:chapter_count]
quota = max(800, int(manifest.get("target_words", 30000)) // len(selected))
sections = "\n".join(f"- {item}" for item in method.framework_sections)
axes = "".join(method.task_axes)
material_text = render_material_inventory(manifest.get("material_inventory") or [])
lines = [
f"# {manifest.get('report_title') or manifest['topic']}:研究框架",
"",
f"research_method: {method.key}",
f"method_name: {method.name}",
f"work_language: 中文主写作;检索关键词、证据摘录、source title、raw notes 可保留英文。",
f"target_words: {manifest.get('target_words', 30000)}",
"",
"## 方法选择",
"",
f"本项目采用 `{method.key}`,因为其结构原则是:{method.structure_principle}",
"",
"框架模块:",
sections,
"",
"Phase 2 任务轴:",
f"- {axes}",
"",
"## 输入材料与使用边界",
"",
material_text,
"",
"这些材料作为现场问题线索和内部事实起点使用;正式结论仍需结合 NMPA、FDA、EMA、ICH、WHO 等权威法规、指南和最佳实践进行验证。",
"",
"## 中心假设",
"",
f"围绕“{manifest['topic']}”形成可被证据支持或证伪的中文主线;所有核心判断必须绑定来源 ID。",
"",
]
for idx, title in enumerate(selected, start=1):
lines.extend(
[
f"## 第{idx}{title}",
"",
f"建议字数:约 {quota} 字。",
f"研究思路:围绕 `{method.key}` 的方法框架,从 {axes} 等任务轴并发收集 evidence packet,再由 chapter assembly 收束为完整中文章节。",
"证据要求:至少 2 个独立 Tier 1-2 信源;不足时在正文标注待验证;必须包含反方证据。",
"",
]
)
lines.extend(
[
"## 暂停点",
"",
"请先确认 `phase1/material_brief.md` 的材料解读和访谈问题,再确认本框架后进入 Phase 2。若章节逻辑、方法框架或字数配额需要调整,应先修改本文件。",
"",
"确认后运行:`uv run python scripts/dr.py approve <project>`;未批准时 `research` 默认会拒绝推进,可用 `--force` 临时覆盖。",
"",
]
)
out = project_root / "phase1" / "framework.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
manifest["phase1"] = {
"status": "completed",
"approved": False,
"framework_path": "phase1/framework.md",
"research_method": method.key,
"updated_at": utc_now_iso(),
}
manifest["updated_at"] = utc_now_iso()
write_manifest(project_root, manifest)
return out
+157
View File
@@ -0,0 +1,157 @@
"""Deterministic Phase 3 review checks for the Python core."""
from __future__ import annotations
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from scripts.runtime.artifacts import load_manifest, write_manifest
from scripts.runtime.tasks import validate_packet
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def _source_ids_from_jsonl(path: Path) -> set[str]:
ids: set[str] = set()
if not path.exists():
return ids
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
source_id = obj.get("id") or obj.get("source_id")
if source_id:
ids.add(str(source_id))
return ids
def _draft_citations(drafts: list[Path]) -> set[str]:
cited: set[str] = set()
for draft in drafts:
cited.update(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", draft.read_text(encoding="utf-8")))
return cited
def _ready_packet_stems(project_root: Path) -> set[str]:
ready: set[str] = set()
for path in sorted((project_root / "phase2" / "packets").glob("*.json")):
try:
packet = json.loads(path.read_text(encoding="utf-8"))
validate_packet(packet)
except Exception:
continue
ready.add(path.stem)
return ready
def _draft_quality_findings(drafts: list[Path]) -> list[dict[str, Any]]:
findings: list[dict[str, Any]] = []
generic_markers = [
"需要进一步完善",
"应当加强",
"持续改进",
"系统性",
"闭环管理",
"质量文化",
]
for draft in drafts:
text = draft.read_text(encoding="utf-8")
zh_chars = sum(1 for char in text if "\u4e00" <= char <= "\u9fff")
citations = re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", text)
evidence_table_present = "证据落点" in text and "待补证据" in text
if zh_chars >= 1200 and len(set(citations)) < 5:
findings.append({"severity": "P1", "message": f"{draft.name} 引用来源过少,可能未充分使用 evidence packet。"})
if zh_chars >= 1200 and not evidence_table_present:
findings.append({"severity": "P1", "message": f"{draft.name} 缺少“证据落点与待补证据”小节,难以判断 evidence 是否真正落到纸面。"})
generic_count = sum(text.count(marker) for marker in generic_markers)
if zh_chars >= 1200 and generic_count >= 18:
findings.append({"severity": "P1", "message": f"{draft.name} 泛化管理表述过多,需要回炉为具体审计发现、风险影响和整改动作。"})
return findings
def build_phase3_critique(project_root: Path) -> Path:
manifest = load_manifest(project_root)
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
ready_stems = _ready_packet_stems(project_root)
packet_errors = [
path for path in sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
if path.stem not in ready_stems
]
chapter_errors = sorted((project_root / "phase2" / "chapter_errors").glob("*.json"))
sources = _source_ids_from_jsonl(project_root / "phase2" / "sources.jsonl")
cited = _draft_citations(drafts)
missing_sources = sorted(cited - sources) if sources else sorted(cited)
uncited_sources = sorted(sources - cited) if cited else sorted(sources)
findings: list[dict[str, Any]] = []
if not drafts:
findings.append({"severity": "P1", "message": "Phase 2 drafts 缺失,尚不能进入 Phase 4 成稿。"})
if packet_errors:
findings.append({"severity": "P1", "message": f"存在 {len(packet_errors)} 个 packet 失败,需要回炉补证据。"})
if chapter_errors:
findings.append({"severity": "P1", "message": f"存在 {len(chapter_errors)} 个章节组装失败,需要修复引用或重写该章。"})
quality_holds = manifest.get("quality_holds") or []
if quality_holds:
findings.append({"severity": "P1", "message": "存在质量暂停标记:" + ", ".join(quality_holds)})
findings.extend(_draft_quality_findings(drafts))
if missing_sources:
findings.append({"severity": "P1", "message": f"正文引用未在 sources.jsonl 中登记:{', '.join(missing_sources)}"})
if not findings:
findings.append({"severity": "P2", "message": "基础产物完整;仍需人工或大上下文模型审校逻辑链、反方证据和章节叙事。"})
lines = [
"# Phase 3 审校 critique",
"",
f"- 项目:{manifest.get('topic', project_root.name)}",
f"- 运行时:python-core-v0.20",
f"- drafts{len(drafts)}",
f"- packets{len(packets)}",
f"- sources{len(sources)}",
f"- cited_source_ids{', '.join(sorted(cited)) if cited else ''}",
"",
"## Findings",
"",
]
for item in findings:
lines.append(f"- [{item['severity']}] {item['message']}")
lines.extend(
[
"",
"## Residual Risks",
"",
"- 本 deterministic review 只做结构、引用和错误包检查;深层逻辑审校仍建议交给 `phase3_review` 角色执行。",
"- 若 sources 为空,本审校会把所有正文引用视为待登记来源。",
"",
"## Next",
"",
"- 若存在 P1,先回到 Phase 2 修复 packet/chapter 错误。",
"- 若仅有 P2,可进入 `dr.py finalize` 的中文原生成稿路径。",
"",
]
)
out = project_root / "phase3" / "critique.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
phase3 = manifest.setdefault("phase3", {})
phase3.update(
{
"status": "completed",
"critique_path": "phase3/critique.md",
"findings_total": len(findings),
"missing_sources": missing_sources,
"uncited_sources": uncited_sources,
"updated_at": utc_now_iso(),
}
)
manifest["updated_at"] = utc_now_iso()
write_manifest(project_root, manifest)
return out
+111
View File
@@ -0,0 +1,111 @@
"""Runtime role and task-model resolution."""
from __future__ import annotations
from dataclasses import dataclass
from scripts.lib.model_config import resolve_model_profile
ROLE_DEFAULTS = {
"dr_plan": {
"skills": ["document-ingest", "search-gateway", "search-strategy", "source-quality", "length-budget", "mckinsey-method"],
"temperature": 0.4,
"max_tokens": 12000,
"max_concurrency": 1,
},
"dr_pm": {
"skills": ["length-budget", "evidence-table", "mckinsey-method"],
"temperature": 0.2,
"max_tokens": 8000,
"max_concurrency": 1,
},
"dr_searcher": {
"skills": ["search-gateway", "search-strategy", "source-quality"],
"temperature": 0.1,
"max_tokens": 6000,
"max_concurrency": 6,
},
"dr_analyst": {
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table", "mckinsey-method"],
"temperature": 0.3,
"max_tokens": 14000,
"max_concurrency": 6,
},
"dr_verifier": {
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table"],
"temperature": 0.2,
"max_tokens": 10000,
"max_concurrency": 4,
},
"dr_chief_editor": {
"skills": ["mckinsey-method", "evidence-table", "output-hygiene"],
"temperature": 0.2,
"max_tokens": 16000,
"max_concurrency": 1,
},
"dr_editor_in_chief": {
"skills": ["mckinsey-method", "citation-manager", "humanizer-cn", "output-hygiene"],
"temperature": 0.4,
"max_tokens": 20000,
"max_concurrency": 1,
},
"dr_reporter": {
"skills": ["pdf-reportlab", "citation-manager", "output-hygiene"],
"temperature": 0.1,
"max_tokens": 6000,
"max_concurrency": 1,
},
}
@dataclass(frozen=True)
class RoleDefinition:
name: str
model: str
skills: list[str]
temperature: float
max_tokens: int
max_concurrency: int
class RuntimeProfile:
def __init__(self, *, profile: str, roles: dict[str, RoleDefinition], task_types: dict[str, str]) -> None:
self.profile = profile
self.roles = roles
self.task_types = task_types
def role_for_task(self, task_type: str) -> RoleDefinition:
role_name = self.task_types.get(task_type)
if not role_name:
raise KeyError(f"unknown task_type: {task_type}")
if role_name not in self.roles:
raise KeyError(f"task_type {task_type} maps to missing role {role_name}")
return self.roles[role_name]
def resolve_runtime_profile(
*,
profile: str | None = None,
overrides: dict[str, str] | None = None,
) -> RuntimeProfile:
resolved = resolve_model_profile(profile=profile, overrides=overrides)
role_models = resolved["roles"]
roles: dict[str, RoleDefinition] = {}
for name, defaults in ROLE_DEFAULTS.items():
model = role_models.get(name)
if not model:
continue
roles[name] = RoleDefinition(
name=name,
model=model,
skills=list(defaults["skills"]),
temperature=float(defaults["temperature"]),
max_tokens=int(defaults["max_tokens"]),
max_concurrency=int(defaults["max_concurrency"]),
)
return RuntimeProfile(
profile=resolved["profile"],
roles=roles,
task_types=dict(resolved.get("task_types") or {}),
)
+107
View File
@@ -0,0 +1,107 @@
"""Canonical skill registry and adapter sync helpers."""
from __future__ import annotations
import shutil
from dataclasses import dataclass
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
CANONICAL_SKILLS_DIR = REPO_ROOT / ".agents" / "skills"
PROJECT_SKILLS_DIR = REPO_ROOT / "skills"
REQUIRED_SKILLS = {
"search-strategy",
"search-gateway",
"source-quality",
"length-budget",
"evidence-table",
"citation-manager",
"mckinsey-method",
}
@dataclass(frozen=True)
class SkillInfo:
name: str
path: Path
class SkillRegistry:
"""Reads skills from the canonical cross-adapter registry."""
def __init__(self, canonical_dir: Path | None = None) -> None:
self.canonical_dir = canonical_dir or CANONICAL_SKILLS_DIR
def roots(self) -> list[Path]:
roots = [self.canonical_dir]
if self.canonical_dir == CANONICAL_SKILLS_DIR and PROJECT_SKILLS_DIR.exists():
roots.append(PROJECT_SKILLS_DIR)
return roots
def list(self) -> list[SkillInfo]:
seen: set[str] = set()
out: list[SkillInfo] = []
for root in self.roots():
if not root.exists():
continue
for path in sorted(root.glob("*/SKILL.md")):
name = path.parent.name
if name in seen:
continue
seen.add(name)
out.append(SkillInfo(name=name, path=path))
return out
def list_names(self) -> list[str]:
return [item.name for item in self.list()]
def read(self, name: str) -> str:
for root in self.roots():
path = root / name / "SKILL.md"
if path.exists():
return path.read_text(encoding="utf-8")
raise FileNotFoundError(f"skill not found: {name}")
def validate(self, required: set[str] | None = None) -> dict[str, object]:
names = set(self.list_names())
required_names = required or REQUIRED_SKILLS
missing = sorted(required_names - names)
malformed: list[str] = []
for item in self.list():
text = item.path.read_text(encoding="utf-8")
if "name:" not in text[:300]:
malformed.append(item.name)
return {
"ok": not missing and not malformed,
"canonical_dir": str(self.canonical_dir),
"count": len(names),
"missing": missing,
"malformed": malformed,
}
def sync_to(self, targets: list[Path], *, force: bool = True) -> int:
"""Copy canonical skills into adapter skill directories.
Returns the number of skill directories copied across all targets.
"""
copied = 0
for target in targets:
if target.resolve() == self.canonical_dir.resolve():
continue
target.mkdir(parents=True, exist_ok=True)
for item in self.list():
dst = target / item.name
if dst.exists() and force:
shutil.rmtree(dst)
if not dst.exists():
shutil.copytree(item.path.parent, dst)
copied += 1
return copied
def default_adapter_skill_dirs() -> list[Path]:
return [
REPO_ROOT / ".opencode" / "skills",
REPO_ROOT / ".agents" / "skills",
]
+65
View File
@@ -0,0 +1,65 @@
"""Source registry helpers for Phase 2 packets."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
def _source_key(source: dict[str, Any]) -> str:
return (source.get("url") or source.get("doi") or source.get("id") or "").strip()
def append_packet_sources(sources_path: Path, packet: dict[str, Any]) -> int:
"""Append packet sources to sources.jsonl, deduping by URL/DOI/id."""
sources_path.parent.mkdir(parents=True, exist_ok=True)
existing: set[str] = set()
if sources_path.exists():
for line in sources_path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
existing.add(_source_key(json.loads(line)))
except json.JSONDecodeError:
continue
written = 0
with sources_path.open("a", encoding="utf-8") as f:
for source in packet.get("sources") or []:
key = _source_key(source)
if not key or key in existing:
continue
existing.add(key)
f.write(json.dumps(source, ensure_ascii=False) + "\n")
written += 1
return written
def rebuild_sources_from_packets(project_root: Path) -> int:
"""Rebuild phase2/sources.jsonl from packet-level source metadata."""
packets_dir = project_root / "phase2" / "packets"
sources_path = project_root / "phase2" / "sources.jsonl"
sources_path.parent.mkdir(parents=True, exist_ok=True)
seen: set[str] = set()
rows: list[dict[str, Any]] = []
for packet_path in sorted(packets_dir.glob("*.json")):
try:
packet = json.loads(packet_path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
continue
for source in packet.get("sources") or []:
if not isinstance(source, dict):
continue
key = _source_key(source)
if not key or key in seen:
continue
seen.add(key)
rows.append(source)
sources_path.write_text(
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
encoding="utf-8",
)
return len(rows)
+229
View File
@@ -0,0 +1,229 @@
"""Task-card and evidence-packet primitives for v0.20 Phase 2."""
from __future__ import annotations
import json
import re
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
from scripts.runtime.methods import ResearchMethod
VALID_ROUTES = {"general", "scholar", "patents", "news"}
DEFAULT_AXES = ["literature", "regulatory", "patents", "market", "counter"]
AXIS_ROUTES = {
"literature": ["scholar", "general"],
"clinical": ["scholar", "general"],
"regulatory": ["general", "news"],
"patents": ["patents", "general"],
"market": ["news", "general"],
"china": ["news", "general"],
"counter": ["scholar", "general"],
"regulatory_gap": ["general", "news"],
"risk_classification": ["general", "scholar"],
"capa_design": ["general", "news"],
"ownership_timeline": ["general"],
"verification_evidence": ["general", "scholar"],
"process_flow": ["scholar", "general"],
"cqa_cpp": ["scholar", "general"],
"scale_up_risk": ["scholar", "general"],
"control_strategy": ["scholar", "general"],
"supply_chain": ["news", "general"],
"scientific_rationale": ["scholar", "general"],
"poc_evidence": ["scholar", "general"],
"ip_fto": ["patents", "general"],
"development_path": ["scholar", "general"],
"commercial_window": ["news", "general"],
"current_state": ["general"],
"capability_gap": ["general"],
"operating_model": ["general"],
"governance": ["general"],
"implementation_roadmap": ["general"],
}
@dataclass
class Chapter:
chapter_id: str
index: int
title: str
notes: str = ""
@dataclass
class TaskCard:
task_id: str
chapter_ids: list[str]
topic_axis: str
questions: list[str]
search_routes: list[str]
output_packet: str
preferred_model_role: str = "dr_analyst"
status: str = "pending"
dependencies: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def parse_framework_chapters(framework_text: str) -> list[Chapter]:
"""Extract Chinese or English chapter headings from a framework markdown."""
lines = framework_text.splitlines()
chapters: list[Chapter] = []
current: Chapter | None = None
note_lines: list[str] = []
heading_re = re.compile(
r"^#{1,3}\s*(?:第\s*)?(\d{1,2})\s*(?:章|[.)、:-])?\s*(.+?)\s*$",
re.IGNORECASE,
)
english_re = re.compile(r"^#{1,3}\s*chapter\s+(\d{1,2})[:.)\s-]+(.+?)\s*$", re.IGNORECASE)
for line in lines:
match = heading_re.match(line.strip()) or english_re.match(line.strip())
if match:
if current:
current.notes = "\n".join(note_lines).strip()
chapters.append(current)
index = int(match.group(1))
title = match.group(2).strip(" #")
current = Chapter(chapter_id=f"ch{index:02d}", index=index, title=title)
note_lines = []
elif current:
note_lines.append(line)
if current:
current.notes = "\n".join(note_lines).strip()
chapters.append(current)
return chapters
def _questions_for_axis(chapter: Chapter, axis: str) -> list[str]:
return [
f"围绕《{chapter.title}》从 {axis} 角度提炼可证伪的核心结论。",
"至少寻找两个 Tier 1-2 来源支撑主要结论;不足时标注待验证。",
"主动检索反方证据、限制条件或失败案例。",
]
def generate_task_cards(
slug: str,
framework_text: str,
*,
axes: list[str] | None = None,
method: ResearchMethod | None = None,
) -> list[TaskCard]:
del slug # slug is kept for call-site clarity and future namespacing.
chapters = parse_framework_chapters(framework_text)
selected_axes = axes or (method.task_axes if method else DEFAULT_AXES)
cards: list[TaskCard] = []
for chapter in chapters:
for axis in selected_axes:
routes = AXIS_ROUTES.get(axis, ["general"])
cards.append(
TaskCard(
task_id=f"{chapter.chapter_id}-{axis}",
chapter_ids=[chapter.chapter_id],
topic_axis=axis,
questions=_questions_for_axis(chapter, axis),
search_routes=routes,
output_packet=f"phase2/packets/{chapter.chapter_id}-{axis}.json",
preferred_model_role="dr_verifier" if axis == "counter" else "dr_analyst",
)
)
validate_task_cards(cards)
return cards
def detect_dependency_cycles(cards: list[TaskCard]) -> None:
graph = {card.task_id: card.dependencies for card in cards}
visiting: set[str] = set()
visited: set[str] = set()
def visit(node: str) -> None:
if node in visiting:
raise ValueError(f"dependency cycle detected at {node}")
if node in visited:
return
visiting.add(node)
for dep in graph.get(node, []):
visit(dep)
visiting.remove(node)
visited.add(node)
for task_id in graph:
visit(task_id)
def validate_task_cards(cards: list[TaskCard]) -> None:
seen: set[str] = set()
for card in cards:
if card.task_id in seen:
raise ValueError(f"duplicate task_id: {card.task_id}")
seen.add(card.task_id)
if not card.chapter_ids:
raise ValueError(f"{card.task_id}: chapter_ids required")
if not card.questions:
raise ValueError(f"{card.task_id}: questions required")
if not card.output_packet.endswith(".json"):
raise ValueError(f"{card.task_id}: output_packet must be json")
invalid_routes = sorted(set(card.search_routes) - VALID_ROUTES)
if invalid_routes:
raise ValueError(f"{card.task_id}: invalid search_routes {invalid_routes}")
missing_deps = sorted({dep for card in cards for dep in card.dependencies} - seen)
if missing_deps:
raise ValueError(f"unknown dependencies: {missing_deps}")
detect_dependency_cycles(cards)
def write_task_cards(path: Path, cards: list[TaskCard]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps([card.to_dict() for card in cards], ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
def load_task_cards(path: Path) -> list[TaskCard]:
data = json.loads(path.read_text(encoding="utf-8"))
cards = [TaskCard(**item) for item in data]
validate_task_cards(cards)
return cards
def validate_packet(packet: dict[str, Any]) -> None:
required = {
"task_id",
"claims",
"evidence_items",
"counter_evidence",
"source_ids",
"source_quality_notes",
"open_questions",
"raw_quotes_or_notes",
}
missing = sorted(required - set(packet))
if missing:
raise ValueError(f"packet missing fields: {missing}")
if not packet["claims"]:
raise ValueError("packet claims must not be empty")
if not packet["evidence_items"]:
raise ValueError("packet evidence_items must not be empty")
if not packet["counter_evidence"]:
raise ValueError("packet counter_evidence must not be empty")
declared = set(packet.get("source_ids") or [])
referenced: set[str] = set()
for section in ("claims", "counter_evidence"):
for item in packet.get(section) or []:
referenced.update(item.get("source_ids") or [])
for item in packet.get("evidence_items") or []:
if item.get("source_id"):
referenced.add(item["source_id"])
undeclared = sorted(referenced - declared)
if undeclared:
raise ValueError(f"packet source_ids referenced but not declared: {undeclared}")
packet_sources = packet.get("sources") or []
if packet_sources:
known_source_ids = {source.get("id") for source in packet_sources}
missing_sources = sorted(declared - known_source_ids)
if missing_sources:
raise ValueError(f"packet source_ids missing source metadata: {missing_sources}")
+261
View File
@@ -0,0 +1,261 @@
"""Python role workers for task-card execution."""
from __future__ import annotations
import json
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Callable, Protocol, Any
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
from scripts.runtime.skills import SkillRegistry
from scripts.runtime.tasks import TaskCard, validate_packet
from scripts.runtime.sources import append_packet_sources
class ChatClient(Protocol):
def chat_complete(self, **kwargs) -> str:
...
class SearchProvider(Protocol):
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
...
class ProjectSearchProvider:
"""Thin adapter over the project-owned search client."""
def __init__(self, *, strict_specialized: bool = True) -> None:
from scripts.lib.search_client import SearchClient
self.client = SearchClient(strict_specialized=strict_specialized)
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
if route == "scholar":
hits = self.client.scholar(query, num_results=num_results, year_low=2020)
elif route == "patents":
hits = self.client.patents(query, num_results=num_results)
elif route == "news":
hits = self.client.news(query, num_results=num_results, time_range="y")
else:
hits = self.client.search(query, num_results=num_results)
return [
{
"title": hit.title,
"url": hit.url,
"snippet": hit.snippet,
"route": route,
}
for hit in hits
]
def close(self) -> None:
self.client.close()
def _extract_json_object(text: str) -> dict:
stripped = text.strip()
if stripped.startswith("```"):
stripped = stripped.strip("`")
if stripped.startswith("json"):
stripped = stripped[4:].strip()
start = stripped.find("{")
end = stripped.rfind("}")
if start == -1 or end == -1 or end < start:
raise ValueError("worker response does not contain a JSON object")
return json.loads(stripped[start : end + 1])
def _safe_source_stem(task_id: str) -> str:
return re.sub(r"[^a-zA-Z0-9]+", "_", task_id).strip("_").lower()
def build_search_context(
card: TaskCard,
search_provider: SearchProvider,
*,
num_results_per_route: int = 5,
) -> dict[str, Any]:
candidate_sources: list[dict[str, Any]] = []
routes_used: list[str] = []
source_stem = _safe_source_stem(card.task_id)
idx = 1
query = " ".join(card.questions)
for route in card.search_routes:
routes_used.append(route)
hits = search_provider.search(query=query, route=route, num_results=num_results_per_route)
for hit in hits:
candidate_sources.append(
{
"id": f"src_{source_stem}_{idx:03d}",
"title": hit.get("title", ""),
"url": hit.get("url", ""),
"snippet": hit.get("snippet", ""),
"route": hit.get("route", route),
"tier": "Tier 2",
"score": 6,
}
)
idx += 1
return {"routes_used": routes_used, "candidate_sources": candidate_sources}
def build_packet_user_prompt(card: TaskCard, search_context: dict[str, Any] | None = None) -> str:
context = search_context or {"routes_used": [], "candidate_sources": []}
return (
"请根据以下 task card 产出一个证据包 JSON。\n"
"正式结论、summary、open_questions 用中文;英文原文摘录、source title、DOI/URL 可以保留英文。\n"
"必须主动包含 counter_evidence,且所有引用的 source_id 必须出现在 source_ids 中。\n\n"
"只能使用 candidate_sources 中的来源,不得编造 URL、DOI、trial ID 或 source_id。\n"
"输出 JSON 必须包含 sources 字段,且 sources 只能来自 candidate_sources。\n\n"
f"{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
"只输出 JSON,不要输出 Markdown 解释。"
)
def build_packet_repair_prompt(
*,
card: TaskCard,
raw_response: str,
error: Exception,
search_context: dict[str, Any] | None = None,
) -> str:
context = search_context or {"routes_used": [], "candidate_sources": []}
return (
"请修复上一次 evidence packet 输出,使其成为合法且通过 schema 校验的 JSON。\n"
"只输出 JSON 对象,不要输出 Markdown、解释或代码块。\n"
"保留中文主写作;英文只允许出现在来源标题、URL、DOI、原文摘录或检索笔记中。\n"
"不得编造 candidate_sources 以外的来源、URL、DOI、trial ID 或 source_id。\n\n"
f"Schema error:\n{error}\n\n"
f"Task card:\n{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
f"Previous raw response:\n{raw_response[:12000]}"
)
class PacketWorker:
def __init__(
self,
*,
role: RoleDefinition,
client: ChatClient,
search_provider: SearchProvider | None = None,
skill_registry: SkillRegistry | None = None,
num_results_per_route: int = 5,
) -> None:
self.role = role
self.client = client
self.search_provider = search_provider
self.skill_registry = skill_registry or SkillRegistry()
self.num_results_per_route = num_results_per_route
def _system_prompt(self) -> str:
skill_texts = []
for name in self.role.skills:
try:
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
except FileNotFoundError:
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
return (
"你是 Deep Research v0.20 Python runtime 的证据包 worker。\n"
"你的唯一任务是把一个 task card 转换为结构化 evidence packet。\n"
"遵循中文主写作原则;不要写章节正文;不要编造 URL、DOI、trial ID 或 source_id。\n\n"
"搜索只能走项目 Python search gateway 或调用方提供的 search_context;不要直接使用 Tavily MCP、browser MCP、平台 web search 或任何需要用户权限确认的外部搜索工具。\n\n"
+ "\n\n".join(skill_texts)
)
def run(self, card: TaskCard) -> dict:
search_context = None
if self.search_provider:
search_context = build_search_context(
card,
self.search_provider,
num_results_per_route=self.num_results_per_route,
)
raw = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_packet_user_prompt(card, search_context),
temperature=self.role.temperature,
max_tokens=self.role.max_tokens,
tag=f"packet:{card.task_id}",
)
try:
packet = _extract_json_object(raw)
validate_packet(packet)
return packet
except Exception as error:
repaired = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_packet_repair_prompt(
card=card,
raw_response=raw,
error=error,
search_context=search_context,
),
temperature=0,
max_tokens=self.role.max_tokens,
tag=f"packet-repair:{card.task_id}",
)
packet = _extract_json_object(repaired)
validate_packet(packet)
return packet
def _write_packet_error(project_root: Path, card: TaskCard, error: Exception) -> None:
path = project_root / "phase2" / "packet_errors" / f"{card.task_id}.json"
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"task_id": card.task_id,
"status": "failed",
"error": str(error),
"output_packet": card.output_packet,
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def run_packet_workers(
*,
project_root: Path,
cards: list[TaskCard],
runtime: RuntimeProfile,
client_factory: Callable[[RoleDefinition], ChatClient],
search_provider_factory: Callable[[], SearchProvider] | None = None,
workers: int,
) -> int:
role = runtime.role_for_task("evidence_packet")
max_workers = max(1, min(workers, role.max_concurrency))
def run_one(card: TaskCard) -> tuple[TaskCard, dict | None, Exception | None]:
search_provider = search_provider_factory() if search_provider_factory else None
try:
worker = PacketWorker(role=role, client=client_factory(role), search_provider=search_provider)
return card, worker.run(card), None
except Exception as error:
return card, None, error
finally:
close = getattr(search_provider, "close", None)
if close:
close()
written = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = [pool.submit(run_one, card) for card in cards]
for future in as_completed(futures):
card, packet, error = future.result()
if error is not None:
_write_packet_error(project_root, card, error)
continue
if packet is None:
_write_packet_error(project_root, card, RuntimeError("packet worker returned no packet"))
continue
path = project_root / card.output_packet
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
append_packet_sources(project_root / "phase2" / "sources.jsonl", packet)
written += 1
return written