v0.20 alpha skill-driven python core
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
"""v0.20 Python runtime core for Deep Research.
|
||||
|
||||
The runtime layer is intentionally platform-neutral: OpenCode, Codex, and
|
||||
Claude Code should call into these modules instead of owning orchestration.
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Project artifact helpers shared by the Python runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PROJECTS_DIR = REPO_ROOT / "projects"
|
||||
|
||||
|
||||
def resolve_project(project: str | Path) -> Path:
|
||||
p = Path(project)
|
||||
if p.is_dir():
|
||||
return p.resolve()
|
||||
candidate = PROJECTS_DIR / str(project)
|
||||
if candidate.is_dir():
|
||||
return candidate.resolve()
|
||||
raise FileNotFoundError(f"project not found: {project}")
|
||||
|
||||
|
||||
def load_manifest(project_root: Path) -> dict[str, Any]:
|
||||
path = project_root / "manifest.json"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"manifest not found: {path}")
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def write_manifest(project_root: Path, manifest: dict[str, Any]) -> None:
|
||||
path = project_root / "manifest.json"
|
||||
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def ensure_phase_dirs(project_root: Path) -> None:
|
||||
for rel in (
|
||||
"phase1",
|
||||
"phase2/drafts",
|
||||
"phase2/evidence",
|
||||
"phase2/packets",
|
||||
"phase3",
|
||||
"phase4",
|
||||
):
|
||||
(project_root / rel).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Chapter brief aggregation and Chinese chapter assembly."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
|
||||
from scripts.runtime.skills import SkillRegistry
|
||||
from scripts.runtime.tasks import load_task_cards, validate_packet
|
||||
from scripts.runtime.workers import ChatClient
|
||||
|
||||
|
||||
def validate_chapter_brief(brief: dict) -> None:
|
||||
required = {
|
||||
"chapter_id",
|
||||
"chapter_title",
|
||||
"packet_ids",
|
||||
"core_claims",
|
||||
"evidence_items",
|
||||
"counter_evidence",
|
||||
"source_ids",
|
||||
"open_questions",
|
||||
"assembly_notes",
|
||||
}
|
||||
missing = sorted(required - set(brief))
|
||||
if missing:
|
||||
raise ValueError(f"chapter brief missing fields: {missing}")
|
||||
if not brief["chapter_id"]:
|
||||
raise ValueError("chapter_id required")
|
||||
if not brief["packet_ids"]:
|
||||
raise ValueError("chapter brief requires at least one packet")
|
||||
if not brief["core_claims"]:
|
||||
raise ValueError("chapter brief requires core_claims")
|
||||
if not brief["evidence_items"]:
|
||||
raise ValueError("chapter brief requires evidence_items")
|
||||
if not brief["counter_evidence"]:
|
||||
raise ValueError("chapter brief requires counter_evidence")
|
||||
|
||||
|
||||
def validate_chapter_markdown_citations(markdown: str, brief: dict) -> None:
|
||||
validate_chapter_brief(brief)
|
||||
cited = set(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", markdown))
|
||||
allowed = set(brief.get("source_ids") or [])
|
||||
unknown = sorted(cited - allowed)
|
||||
if unknown:
|
||||
raise ValueError(f"unknown citation ids in {brief['chapter_id']}: {unknown}")
|
||||
|
||||
|
||||
def _chapter_title_from_id(chapter_id: str) -> str:
|
||||
try:
|
||||
index = int(chapter_id.replace("ch", ""))
|
||||
return f"第{index}章"
|
||||
except ValueError:
|
||||
return chapter_id
|
||||
|
||||
|
||||
def build_chapter_briefs(project_root: Path) -> list[dict]:
|
||||
cards = load_task_cards(project_root / "phase2" / "task_cards.json")
|
||||
grouped: dict[str, list[tuple[str, dict]]] = {}
|
||||
for card in cards:
|
||||
packet_path = project_root / card.output_packet
|
||||
if not packet_path.exists():
|
||||
continue
|
||||
packet = json.loads(packet_path.read_text(encoding="utf-8"))
|
||||
validate_packet(packet)
|
||||
for chapter_id in card.chapter_ids:
|
||||
grouped.setdefault(chapter_id, []).append((card.task_id, packet))
|
||||
|
||||
briefs: list[dict] = []
|
||||
out_dir = project_root / "phase2" / "chapter_briefs"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for chapter_id in sorted(grouped):
|
||||
packet_pairs = sorted(grouped[chapter_id], key=lambda item: item[0])
|
||||
packet_ids = [item[0] for item in packet_pairs]
|
||||
packets = [item[1] for item in packet_pairs]
|
||||
source_ids = sorted({sid for packet in packets for sid in packet.get("source_ids", [])})
|
||||
brief = {
|
||||
"chapter_id": chapter_id,
|
||||
"chapter_title": _chapter_title_from_id(chapter_id),
|
||||
"packet_ids": packet_ids,
|
||||
"core_claims": [claim for packet in packets for claim in packet.get("claims", [])],
|
||||
"evidence_items": [item for packet in packets for item in packet.get("evidence_items", [])],
|
||||
"counter_evidence": [item for packet in packets for item in packet.get("counter_evidence", [])],
|
||||
"source_ids": source_ids,
|
||||
"open_questions": [q for packet in packets for q in packet.get("open_questions", [])],
|
||||
"assembly_notes": [
|
||||
"用中文写正式章节,英文仅保留在必要的来源标题、原文摘录、DOI/URL 中。",
|
||||
"避免碎片化:不要按 packet 逐段堆砌,要先提炼本章主线,再组织证据。",
|
||||
"每个事实、数字和关键判断都必须保留 [src_xxx] 引用。",
|
||||
"必须纳入 counter_evidence,并说明它如何影响结论置信度。",
|
||||
],
|
||||
}
|
||||
validate_chapter_brief(brief)
|
||||
(out_dir / f"{chapter_id}.json").write_text(
|
||||
json.dumps(brief, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
briefs.append(brief)
|
||||
return briefs
|
||||
|
||||
|
||||
def build_chapter_user_prompt(brief: dict) -> str:
|
||||
return (
|
||||
"请根据以下 chapter brief 写一章正式中文 Markdown 正文。\n"
|
||||
"目标是形成一个完整章节,而不是 packet 摘要。避免碎片化,按金字塔结构组织:章首先给结论,再用证据支撑。\n"
|
||||
"要求:标题必须是观点型判断;每个数字和事实保留 [src_xxx];纳入反方证据;不要出现调度元数据。\n"
|
||||
"禁止写空泛咨询腔。每个二级小节都必须至少落下 2 个具体审计发现、法规要求、SOP/记录/参数/现场观察或整改证据;不要只写原则。\n"
|
||||
"正文末尾必须增加“证据落点与待补证据”小节,用表格列出:关键判断、已使用证据 source_id、已落地整改动作、仍缺证据。若证据不足,直接标注需回炉 Phase 2,不要用泛泛表述补齐。\n"
|
||||
"只输出 Markdown,不要输出解释。\n\n"
|
||||
f"{json.dumps(brief, ensure_ascii=False, indent=2)}"
|
||||
)
|
||||
|
||||
|
||||
class ChapterAssemblyWorker:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
role: RoleDefinition,
|
||||
client: ChatClient,
|
||||
skill_registry: SkillRegistry | None = None,
|
||||
) -> None:
|
||||
self.role = role
|
||||
self.client = client
|
||||
self.skill_registry = skill_registry or SkillRegistry()
|
||||
|
||||
def _system_prompt(self) -> str:
|
||||
skill_texts = []
|
||||
for name in self.role.skills:
|
||||
try:
|
||||
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
|
||||
except FileNotFoundError:
|
||||
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
|
||||
return (
|
||||
"你是 Deep Research v0.20 的中文章节组装 worker。\n"
|
||||
"你的职责是把结构化证据包收束成连贯章节,解决并发研究造成的碎片化。\n"
|
||||
"不得编造来源,不得删除关键反方证据。\n\n"
|
||||
+ "\n\n".join(skill_texts)
|
||||
)
|
||||
|
||||
def write_chapter(self, *, project_root: Path, brief: dict) -> Path:
|
||||
validate_chapter_brief(brief)
|
||||
markdown = self.client.chat_complete(
|
||||
model=self.role.model,
|
||||
system=self._system_prompt(),
|
||||
user=build_chapter_user_prompt(brief),
|
||||
temperature=self.role.temperature,
|
||||
max_tokens=self.role.max_tokens,
|
||||
tag=f"chapter:{brief['chapter_id']}",
|
||||
)
|
||||
validate_chapter_markdown_citations(markdown, brief)
|
||||
out = project_root / "phase2" / "drafts" / f"{brief['chapter_id']}.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(markdown.rstrip() + "\n", encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
def _write_chapter_error(project_root: Path, brief: dict, error: Exception) -> None:
|
||||
path = project_root / "phase2" / "chapter_errors" / f"{brief.get('chapter_id', 'unknown')}.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"chapter_id": brief.get("chapter_id"),
|
||||
"status": "failed",
|
||||
"error": str(error),
|
||||
}
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def run_chapter_assembly_workers(
|
||||
*,
|
||||
project_root: Path,
|
||||
briefs: list[dict],
|
||||
runtime: RuntimeProfile,
|
||||
client_factory: Callable[[RoleDefinition], ChatClient],
|
||||
workers: int,
|
||||
) -> int:
|
||||
role = runtime.role_for_task("chapter_assembly")
|
||||
max_workers = max(1, min(workers, role.max_concurrency))
|
||||
|
||||
def run_one(brief: dict) -> tuple[dict, Path | None, Exception | None]:
|
||||
try:
|
||||
worker = ChapterAssemblyWorker(role=role, client=client_factory(role))
|
||||
return brief, worker.write_chapter(project_root=project_root, brief=brief), None
|
||||
except Exception as error:
|
||||
return brief, None, error
|
||||
|
||||
written = 0
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = [pool.submit(run_one, brief) for brief in briefs]
|
||||
for future in as_completed(futures):
|
||||
brief, path, error = future.result()
|
||||
if error is not None:
|
||||
_write_chapter_error(project_root, brief, error)
|
||||
continue
|
||||
if path is None:
|
||||
_write_chapter_error(project_root, brief, RuntimeError("chapter worker returned no output path"))
|
||||
continue
|
||||
written += 1
|
||||
return written
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Phase 0 user-provided material ingestion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_FIRERED_OCR_ENDPOINT = "http://192.168.50.100:8001"
|
||||
DEFAULT_OCR_MAX_PAGES = 50
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OcrResult:
|
||||
text: str
|
||||
pages_processed: int
|
||||
output_path: Path
|
||||
|
||||
|
||||
def safe_filename(path: Path) -> str:
|
||||
name = path.name.strip()
|
||||
return name or "material"
|
||||
|
||||
|
||||
def extract_pdf_text(path: Path) -> tuple[str, int, bool]:
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(str(path))
|
||||
chunks: list[str] = []
|
||||
for index, page in enumerate(reader.pages, start=1):
|
||||
text = (page.extract_text() or "").strip()
|
||||
if text:
|
||||
chunks.append(f"\n\n## Page {index}\n\n{text}")
|
||||
combined = "".join(chunks).strip()
|
||||
ocr_required = len(combined) < max(20, len(reader.pages) * 20)
|
||||
return combined, len(reader.pages), ocr_required
|
||||
|
||||
|
||||
def ocr_endpoint_from_env() -> str:
|
||||
return os.environ.get("DEEP_RESEARCH_OCR_ENDPOINT", DEFAULT_FIRERED_OCR_ENDPOINT).rstrip("/")
|
||||
|
||||
|
||||
def ocr_max_pages_from_env() -> int:
|
||||
raw = os.environ.get("DEEP_RESEARCH_OCR_MAX_PAGES")
|
||||
if not raw:
|
||||
return DEFAULT_OCR_MAX_PAGES
|
||||
try:
|
||||
return max(1, int(raw))
|
||||
except ValueError:
|
||||
return DEFAULT_OCR_MAX_PAGES
|
||||
|
||||
|
||||
def render_pdf_pages(pdf_path: Path, output_dir: Path, *, max_pages: int) -> list[Path]:
|
||||
import fitz
|
||||
|
||||
pages_dir = output_dir / f"{pdf_path.stem}.ocr-pages"
|
||||
pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
image_paths: list[Path] = []
|
||||
doc = fitz.open(pdf_path)
|
||||
try:
|
||||
for index, page in enumerate(doc[:max_pages], start=1):
|
||||
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
|
||||
image_path = pages_dir / f"page-{index:03d}.png"
|
||||
pix.save(image_path)
|
||||
image_paths.append(image_path)
|
||||
finally:
|
||||
doc.close()
|
||||
return image_paths
|
||||
|
||||
|
||||
def data_url_for_image(path: Path) -> str:
|
||||
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
|
||||
return f"data:image/png;base64,{encoded}"
|
||||
|
||||
|
||||
def call_firered_ocr(image_path: Path, *, endpoint: str) -> str:
|
||||
payload = {
|
||||
"model": "firered-ocr",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "请识别图片中的全部文字,保持原有顺序,只输出文字。"},
|
||||
{"type": "image_url", "image_url": {"url": data_url_for_image(image_path)}},
|
||||
],
|
||||
}
|
||||
],
|
||||
"temperature": 0,
|
||||
"max_tokens": 3000,
|
||||
}
|
||||
response = requests.post(f"{endpoint.rstrip('/')}/v1/chat/completions", json=payload, timeout=60)
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"{response.status_code} {response.text[:500]}")
|
||||
data = response.json()
|
||||
return str(data["choices"][0]["message"].get("content") or "").strip()
|
||||
|
||||
|
||||
def ocr_pdf_with_firered(*, pdf_path: Path, output_dir: Path, endpoint: str, max_pages: int) -> OcrResult:
|
||||
image_paths = render_pdf_pages(pdf_path, output_dir, max_pages=max_pages)
|
||||
chunks: list[str] = []
|
||||
for index, image_path in enumerate(image_paths, start=1):
|
||||
text = call_firered_ocr(image_path, endpoint=endpoint)
|
||||
if text:
|
||||
chunks.append(f"\n\n## OCR Page {index}\n\n{text}")
|
||||
combined = "".join(chunks).strip()
|
||||
output_path = output_dir / f"{pdf_path.stem}.ocr.md"
|
||||
body = [
|
||||
f"# OCR Material: {pdf_path.name}",
|
||||
"",
|
||||
f"- source_path: {pdf_path}",
|
||||
f"- endpoint: {endpoint}",
|
||||
f"- pages_processed: {len(image_paths)}",
|
||||
"",
|
||||
combined or "OCR 未返回可用文本。",
|
||||
"",
|
||||
]
|
||||
output_path.write_text("\n".join(body), encoding="utf-8")
|
||||
return OcrResult(text=combined, pages_processed=len(image_paths), output_path=output_path)
|
||||
|
||||
|
||||
def ingest_input_materials(project_root: Path, materials: list[str] | None) -> list[dict[str, Any]]:
|
||||
inventory: list[dict[str, Any]] = []
|
||||
if not materials:
|
||||
return inventory
|
||||
|
||||
inputs_dir = project_root / "phase0" / "inputs"
|
||||
extracted_dir = project_root / "phase0" / "extracted"
|
||||
ocr_endpoint = ocr_endpoint_from_env()
|
||||
ocr_max_pages = ocr_max_pages_from_env()
|
||||
inputs_dir.mkdir(parents=True, exist_ok=True)
|
||||
extracted_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for raw in materials:
|
||||
source = Path(raw).expanduser()
|
||||
if not source.exists():
|
||||
inventory.append({"kind": "note", "note": raw})
|
||||
continue
|
||||
|
||||
copied = inputs_dir / safe_filename(source)
|
||||
shutil.copy2(source, copied)
|
||||
item: dict[str, Any] = {
|
||||
"kind": source.suffix.lower().lstrip(".") or "file",
|
||||
"source_path": str(source),
|
||||
"copied_to": str(copied.relative_to(project_root)),
|
||||
"size_bytes": source.stat().st_size,
|
||||
}
|
||||
|
||||
if source.suffix.lower() == ".pdf":
|
||||
text, pages, ocr_required = extract_pdf_text(source)
|
||||
extracted = extracted_dir / f"{source.stem}.md"
|
||||
ocr_result: OcrResult | None = None
|
||||
ocr_error: str | None = None
|
||||
if ocr_required:
|
||||
try:
|
||||
ocr_result = ocr_pdf_with_firered(
|
||||
pdf_path=source,
|
||||
output_dir=extracted_dir,
|
||||
endpoint=ocr_endpoint,
|
||||
max_pages=min(pages, ocr_max_pages),
|
||||
)
|
||||
if ocr_result.text:
|
||||
text = "\n\n".join(part for part in [text, ocr_result.text] if part)
|
||||
except Exception as exc: # noqa: BLE001 - ingestion should not block project init.
|
||||
ocr_error = str(exc)
|
||||
|
||||
body = [
|
||||
f"# Extracted Material: {source.name}",
|
||||
"",
|
||||
f"- source_path: {source}",
|
||||
f"- copied_to: {copied.relative_to(project_root)}",
|
||||
f"- pages: {pages}",
|
||||
f"- ocr_required: {str(ocr_required).lower()}",
|
||||
f"- ocr_status: {'completed' if ocr_result else 'failed' if ocr_error else 'not_required'}",
|
||||
"",
|
||||
text or "未能从 PDF 直接抽取文本;该材料可能需要 OCR。",
|
||||
"",
|
||||
]
|
||||
if ocr_error:
|
||||
body.extend(["## OCR Error", "", ocr_error, ""])
|
||||
extracted.write_text("\n".join(body), encoding="utf-8")
|
||||
item.update(
|
||||
{
|
||||
"pages": pages,
|
||||
"extracted_to": str(extracted.relative_to(project_root)),
|
||||
"text_chars": len(text),
|
||||
"ocr_required": ocr_required,
|
||||
"ocr_status": "completed" if ocr_result else "failed" if ocr_error else "not_required",
|
||||
}
|
||||
)
|
||||
if ocr_result:
|
||||
item.update(
|
||||
{
|
||||
"ocr_endpoint": ocr_endpoint,
|
||||
"ocr_pages_processed": ocr_result.pages_processed,
|
||||
"ocr_extracted_to": str(ocr_result.output_path.relative_to(project_root)),
|
||||
"ocr_text_chars": len(ocr_result.text),
|
||||
}
|
||||
)
|
||||
if ocr_error:
|
||||
item["ocr_error"] = ocr_error
|
||||
else:
|
||||
item["ocr_required"] = source.suffix.lower() in {".png", ".jpg", ".jpeg", ".tif", ".tiff"}
|
||||
inventory.append(item)
|
||||
|
||||
return inventory
|
||||
|
||||
|
||||
def render_material_inventory(inventory: list[dict[str, Any]]) -> str:
|
||||
if not inventory:
|
||||
return "- 暂无;可通过 `--input-material` 加入审计报告、问题清单或内部记录。"
|
||||
lines: list[str] = []
|
||||
for item in inventory:
|
||||
if item.get("kind") == "note":
|
||||
lines.append(f"- 备注:{item.get('note', '')}")
|
||||
continue
|
||||
marker = ";需要 OCR" if item.get("ocr_required") else ""
|
||||
ocr = f";OCR:{item.get('ocr_status')}" if item.get("ocr_status") else ""
|
||||
extracted = item.get("extracted_to")
|
||||
extra = f";抽取文本:{extracted}" if extracted else ""
|
||||
lines.append(
|
||||
f"- {item.get('copied_to')}({item.get('kind')},{item.get('size_bytes', 0)} bytes{extra}{marker}{ocr})"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Research method registry for Phase 1 framework selection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_METHOD_CONFIG = REPO_ROOT / "configs" / "research_methods.yaml"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResearchMethod:
|
||||
key: str
|
||||
name: str
|
||||
best_for: list[str]
|
||||
structure_principle: str
|
||||
task_axes: list[str]
|
||||
framework_sections: list[str]
|
||||
|
||||
|
||||
class ResearchMethodRegistry:
|
||||
def __init__(self, path: Path | None = None) -> None:
|
||||
self.path = path or DEFAULT_METHOD_CONFIG
|
||||
self._data = self._load()
|
||||
|
||||
def _load(self) -> dict[str, Any]:
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"research method config not found: {self.path}")
|
||||
data = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {}
|
||||
if not isinstance(data, dict) or "methods" not in data:
|
||||
raise ValueError(f"invalid research method config: {self.path}")
|
||||
return data
|
||||
|
||||
@property
|
||||
def default_method(self) -> str:
|
||||
return (self._data.get("defaults") or {}).get("method", "mckinsey_market")
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
return sorted((self._data.get("methods") or {}).keys())
|
||||
|
||||
def get(self, key: str | None = None) -> ResearchMethod:
|
||||
selected = key or self.default_method
|
||||
methods = self._data.get("methods") or {}
|
||||
if selected not in methods:
|
||||
raise KeyError(f"unknown research_method: {selected}")
|
||||
item = methods[selected] or {}
|
||||
return ResearchMethod(
|
||||
key=selected,
|
||||
name=item.get("name", selected),
|
||||
best_for=list(item.get("best_for") or []),
|
||||
structure_principle=item.get("structure_principle", ""),
|
||||
task_axes=list(item.get("task_axes") or []),
|
||||
framework_sections=list(item.get("framework_sections") or []),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Deterministic orchestration helpers for v0.20."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.runtime.artifacts import ensure_phase_dirs, load_manifest, write_manifest
|
||||
from scripts.runtime.methods import ResearchMethodRegistry
|
||||
from scripts.runtime.tasks import generate_task_cards, write_task_cards
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def create_phase2_task_cards(
|
||||
project_root: Path,
|
||||
*,
|
||||
axes: list[str] | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> list[dict[str, object]]:
|
||||
framework = project_root / "phase1" / "framework.md"
|
||||
if not framework.exists():
|
||||
raise FileNotFoundError(f"framework not found: {framework}")
|
||||
method_key = None
|
||||
if (project_root / "manifest.json").exists():
|
||||
method_key = load_manifest(project_root).get("research_method")
|
||||
method = ResearchMethodRegistry().get(method_key)
|
||||
cards = generate_task_cards(project_root.name, framework.read_text(encoding="utf-8"), axes=axes, method=method)
|
||||
if not dry_run:
|
||||
ensure_phase_dirs(project_root)
|
||||
write_task_cards(project_root / "phase2" / "task_cards.json", cards)
|
||||
manifest = load_manifest(project_root)
|
||||
phase2 = manifest.setdefault("phase2", {})
|
||||
phase2.update(
|
||||
{
|
||||
"status": "in_progress",
|
||||
"runtime": "python-core-v0.20",
|
||||
"research_method": method.key,
|
||||
"task_cards_path": "phase2/task_cards.json",
|
||||
"task_cards_total": len(cards),
|
||||
"updated_at": utc_now_iso(),
|
||||
}
|
||||
)
|
||||
write_manifest(project_root, manifest)
|
||||
return [card.to_dict() for card in cards]
|
||||
|
||||
|
||||
def write_placeholder_packets(
|
||||
project_root: Path,
|
||||
task_cards: list[dict[str, object]],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> int:
|
||||
"""Create packet skeletons for manual/API completion.
|
||||
|
||||
This keeps the first v0.20 implementation deterministic and resumable; LLM
|
||||
calls can later fill the same schema without changing downstream readers.
|
||||
"""
|
||||
count = 0
|
||||
for card in task_cards:
|
||||
packet_path = project_root / str(card["output_packet"])
|
||||
packet = {
|
||||
"task_id": card["task_id"],
|
||||
"claims": [],
|
||||
"evidence_items": [],
|
||||
"counter_evidence": [],
|
||||
"source_ids": [],
|
||||
"source_quality_notes": [],
|
||||
"open_questions": ["待由 Python role worker 调用模型补全。"],
|
||||
"raw_quotes_or_notes": [],
|
||||
}
|
||||
if not dry_run:
|
||||
packet_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
packet_path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
count += 1
|
||||
return count
|
||||
@@ -0,0 +1,341 @@
|
||||
"""Phase 1 project initialization and framework generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scripts.runtime.artifacts import PROJECTS_DIR, ensure_phase_dirs, load_manifest, write_manifest
|
||||
from scripts.runtime.materials import ingest_input_materials, render_material_inventory
|
||||
from scripts.runtime.methods import ResearchMethod, ResearchMethodRegistry
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def slugify_topic(topic: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9]+", "-", topic.lower()).strip("-")
|
||||
if slug:
|
||||
return slug[:80]
|
||||
digest = hashlib.sha1(topic.encode("utf-8")).hexdigest()[:8]
|
||||
return f"research-{digest}"
|
||||
|
||||
|
||||
def create_project(
|
||||
*,
|
||||
topic: str,
|
||||
slug: str | None = None,
|
||||
projects_dir: Path = PROJECTS_DIR,
|
||||
method_key: str | None = None,
|
||||
report_type: str = "research",
|
||||
model_profile: str = "medium",
|
||||
target_words: int = 30000,
|
||||
input_materials: list[str] | None = None,
|
||||
) -> Path:
|
||||
method = ResearchMethodRegistry().get(method_key)
|
||||
project_slug = slug or slugify_topic(topic)
|
||||
project_root = projects_dir / project_slug
|
||||
if project_root.exists():
|
||||
raise FileExistsError(f"project already exists: {project_root}")
|
||||
ensure_phase_dirs(project_root)
|
||||
(project_root / "phase0" / "inputs").mkdir(parents=True, exist_ok=True)
|
||||
(project_root / "phase0" / "extracted").mkdir(parents=True, exist_ok=True)
|
||||
material_inventory = ingest_input_materials(project_root, input_materials)
|
||||
now = utc_now_iso()
|
||||
manifest: dict[str, Any] = {
|
||||
"version": "0.20.0",
|
||||
"runtime": "python-core-v0.20",
|
||||
"topic": topic,
|
||||
"slug": project_slug,
|
||||
"report_title": topic,
|
||||
"type": report_type,
|
||||
"work_language": "zh",
|
||||
"model_profile": model_profile,
|
||||
"research_method": method.key,
|
||||
"target_words": target_words,
|
||||
"input_materials": input_materials or [],
|
||||
"material_inventory": material_inventory,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"phase1": {
|
||||
"status": "initialized",
|
||||
"approved": False,
|
||||
"requires_user_interview": True,
|
||||
"material_brief_path": "phase1/material_brief.md",
|
||||
},
|
||||
"phase2": {"status": "pending"},
|
||||
"phase3": {"status": "pending"},
|
||||
"phase4": {"status": "pending"},
|
||||
}
|
||||
write_manifest(project_root, manifest)
|
||||
_write_interview_seed(project_root, manifest, method)
|
||||
write_material_brief(project_root, manifest, method)
|
||||
return project_root
|
||||
|
||||
|
||||
def _write_interview_seed(project_root: Path, manifest: dict[str, Any], method: ResearchMethod) -> None:
|
||||
material_text = render_material_inventory(manifest.get("material_inventory") or [])
|
||||
text = (
|
||||
f"# Phase 1 访谈记录\n\n"
|
||||
f"- 主题:{manifest['topic']}\n"
|
||||
f"- 研究方法:{method.key} - {method.name}\n"
|
||||
f"- 报告类型:{manifest['type']}\n"
|
||||
f"- 目标字数:{manifest['target_words']}\n"
|
||||
f"- 工作语言:中文主写作;检索关键词、证据摘录和来源笔记可保留英文。\n\n"
|
||||
f"## 已提供材料\n\n{material_text}\n\n"
|
||||
"## 后续访谈问题\n\n"
|
||||
"1. 本报告最重要的决策用途是什么?\n"
|
||||
"2. 是否有必须覆盖或必须排除的公司、产品、工艺、市场或法规范围?\n"
|
||||
"3. 结论偏好是战略建议、风险清单、投资判断,还是执行路线图?\n"
|
||||
)
|
||||
(project_root / "phase1" / "interview.md").write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def _material_excerpt(project_root: Path, rel_path: str, *, max_chars: int = 1200) -> str:
|
||||
path = project_root / rel_path
|
||||
if not path.exists():
|
||||
return "(未找到抽取文本)"
|
||||
text = path.read_text(encoding="utf-8")
|
||||
compact = "\n".join(line.rstrip() for line in text.splitlines() if line.strip())
|
||||
return compact[:max_chars] + ("..." if len(compact) > max_chars else "")
|
||||
|
||||
|
||||
def _derive_material_observations(project_root: Path, inventory: list[dict[str, Any]]) -> list[str]:
|
||||
combined_parts: list[str] = []
|
||||
for item in inventory:
|
||||
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
|
||||
if rel and (project_root / rel).exists():
|
||||
combined_parts.append((project_root / rel).read_text(encoding="utf-8"))
|
||||
text = "\n".join(combined_parts)
|
||||
checks = [
|
||||
("审计范围覆盖生产管理、原液、制剂和无菌相关模块,报告需要同时处理 GMP 合规、工艺转移和运营协同,而不是只写质量体系。", ["生产管理", "原液", "制剂", "无菌"]),
|
||||
("材料显示高风险项为 0、中风险项为 1,适合采用“商业化 readiness 与系统成熟度差距”而非“体系失控”作为初始假设。", ["高风险 0", "中风险1", "低风险7"]),
|
||||
("商业化经验、无菌保障细节、文件要求与执行一致性是需要访谈确认的主线风险。", ["商业化经验不足", "无菌保障", "文件要求与执行一致性"]),
|
||||
("工艺规程、批记录、CPP/CQA、VMPR/VMP、验证主计划等内容反复出现,说明工艺验证和商业化文件体系可能是 Phase 2 的重点证据轴。", ["CPP", "CQA", "VMPR", "VMP"]),
|
||||
("温度、压差、WFI、冷却段微生物、RABS/ORABS、first air、APS 等无菌和设施细节需要映射到 EU Annex 1、NMPA GMP 和企业 SOP。", ["温度", "压差", "WFI", "APS"]),
|
||||
("复盘材料包含责任人和局部答复,后续整改路线图应尽量回填 owner、期限、关闭证据和复核机制。", ["填写人", "是否已经回答完整", "整改"]),
|
||||
]
|
||||
observations = [message for message, needles in checks if any(needle in text for needle in needles)]
|
||||
return observations or ["材料已导入但尚未形成足够结构化判断;需要先访谈确认研究用途、范围和优先级。"]
|
||||
|
||||
|
||||
def write_material_brief(
|
||||
project_root: Path,
|
||||
manifest: dict[str, Any] | None = None,
|
||||
method: ResearchMethod | None = None,
|
||||
) -> Path:
|
||||
"""Write a Phase 0/1 material brief that must be reviewed before Phase 2."""
|
||||
manifest = manifest or load_manifest(project_root)
|
||||
method = method or ResearchMethodRegistry().get(manifest.get("research_method"))
|
||||
inventory = manifest.get("material_inventory") or []
|
||||
lines = [
|
||||
f"# Phase 0 材料简报:{manifest.get('topic', project_root.name)}",
|
||||
"",
|
||||
"status: 待用户确认",
|
||||
f"research_method: {method.key}",
|
||||
"",
|
||||
"## 已导入材料",
|
||||
"",
|
||||
render_material_inventory(inventory),
|
||||
"",
|
||||
"## 材料初步解读",
|
||||
"",
|
||||
"以下内容由 Python core 从已落盘材料抽样生成,只作为访谈起点;不得直接视为最终结论。",
|
||||
"",
|
||||
]
|
||||
lines.extend(["## 初步问题聚类(待访谈确认)", ""])
|
||||
for observation in _derive_material_observations(project_root, inventory):
|
||||
lines.append(f"- {observation}")
|
||||
lines.append("")
|
||||
lines.append("## 材料摘录")
|
||||
lines.append("")
|
||||
for item in inventory:
|
||||
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
|
||||
if not rel:
|
||||
continue
|
||||
lines.extend(
|
||||
[
|
||||
f"### {Path(rel).name}",
|
||||
"",
|
||||
_material_excerpt(project_root, rel),
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"## 建议访谈确认点",
|
||||
"",
|
||||
"1. 本报告的最重要用途是什么:内部整改、客户沟通、董事会决策,还是外部审计准备?",
|
||||
"2. 哪些审计发现最需要优先展开:无菌保障、工艺验证、数据完整性、质量体系闭环,还是运营协同?",
|
||||
"3. 是否存在必须排除或脱敏的项目、人员、客户、产品或工艺信息?",
|
||||
"4. 短中长期整改的时间边界如何定义,例如 30/90/180 天,还是按临床/商业化里程碑划分?",
|
||||
"5. 是否需要把 NMPA、FDA、EMA、ICH、WHO 的法规基线分别映射到整改责任人和证据包?",
|
||||
"",
|
||||
"## Gate",
|
||||
"",
|
||||
"请用户确认本材料简报与访谈问题后,再生成或批准 `phase1/framework.md` 并进入 Phase 2。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
out = project_root / "phase1" / "material_brief.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines), encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
CHAPTER_TEMPLATES: dict[str, list[str]] = {
|
||||
"mckinsey_market": [
|
||||
"核心结论先行界定市场机会与约束",
|
||||
"临床与真实世界证据决定需求天花板",
|
||||
"监管路径和支付环境重塑商业化节奏",
|
||||
"竞争格局正在从单点产品转向组合能力",
|
||||
"专利与技术壁垒决定长期利润池",
|
||||
"中国市场的准入和供给能力形成独立变量",
|
||||
"资本市场预期与基本面之间存在可验证偏差",
|
||||
"反方证据限定结论边界并提示回撤风险",
|
||||
"战略选择应围绕资源约束排序",
|
||||
"执行路线图需要把证据缺口转化为行动清单",
|
||||
],
|
||||
"gmp_gap_assessment": [
|
||||
"监管基线决定整改范围而非企业主观偏好",
|
||||
"现状差距需要按法规条款和业务流程双重定位",
|
||||
"质量风险分级决定 CAPA 优先级",
|
||||
"根因分析质量决定整改能否闭环",
|
||||
"CAPA 设计必须绑定责任人、证据和期限",
|
||||
"验证计划决定整改是否可被审计接受",
|
||||
"供应商和外包管理常是系统性缺口放大器",
|
||||
"数据完整性风险需要独立成章处理",
|
||||
"实施路线图需要平衡停线风险与合规风险",
|
||||
"管理层治理机制决定整改能否持续",
|
||||
],
|
||||
"cmc_process_risk": [
|
||||
"工艺流程图是识别放大风险的起点",
|
||||
"CQA 与 CPP 的映射决定控制策略质量",
|
||||
"放大过程的失效模式集中在传质、混合和稳定性",
|
||||
"分析方法和放行标准决定证据可信度",
|
||||
"技术转移风险来自知识隐性化和现场差异",
|
||||
"供应链约束会改变工艺控制边界",
|
||||
"偏差和变更管理决定商业化后的韧性",
|
||||
"监管沟通策略需要提前固化关键假设",
|
||||
"反方证据限定平台工艺可复制性",
|
||||
"CMC 路线图需要把风险转化为验证实验",
|
||||
],
|
||||
"rd_go_no_go": [
|
||||
"科学假设强度决定项目是否值得进入下一阶段",
|
||||
"POC 证据需要同时证明有效性和可转化性",
|
||||
"安全性窗口决定适应症与人群选择",
|
||||
"IP 与 FTO 风险决定商业化自由度",
|
||||
"开发路径需要把关键不确定性前置验证",
|
||||
"竞争窗口决定速度是否仍有战略价值",
|
||||
"CMC 与临床运营能力影响真实可行性",
|
||||
"反方证据决定 go/no-go 阈值",
|
||||
"投资强度应与证据成熟度匹配",
|
||||
"决策门槛需要形成可执行检查表",
|
||||
],
|
||||
"management_consulting": [
|
||||
"现状诊断需要区分症状、根因和约束条件",
|
||||
"能力差距决定组织改进优先级",
|
||||
"流程断点揭示跨部门协作成本",
|
||||
"治理结构决定决策速度和责任清晰度",
|
||||
"运营模型需要匹配战略目标而非照搬标杆",
|
||||
"数字化工具只有嵌入流程才产生价值",
|
||||
"绩效指标需要避免局部最优",
|
||||
"变革阻力本身是方案设计输入",
|
||||
"路线图需要把 quick wins 与系统建设分层",
|
||||
"落地机制决定咨询建议能否转化为成果",
|
||||
],
|
||||
"gmp_quality_operations_diagnosis": [
|
||||
"现场审计发现需要先转化为可验证的系统性问题图谱",
|
||||
"法规基线决定质量体系差距的严重度与整改边界",
|
||||
"生产工艺体系风险来自流程、设施、公用系统和验证证据的耦合缺口",
|
||||
"偏差、变更、CAPA 和数据完整性决定质量系统能否闭环",
|
||||
"人员能力与质量文化决定制度是否真正落地",
|
||||
"运营管理问题需要区分组织、流程、会议机制和指标体系缺口",
|
||||
"跨部门协同断点会放大 GMP 风险和交付风险",
|
||||
"标杆实践应转化为短中长期整改组合而非口号",
|
||||
"整改路线图必须绑定责任、优先级、证据和复核机制",
|
||||
"管理层治理机制决定白帆能否从一次整改转向持续改进",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def render_framework(project_root: Path, *, method_key: str | None = None, chapter_count: int = 10) -> Path:
|
||||
manifest = load_manifest(project_root)
|
||||
registry = ResearchMethodRegistry()
|
||||
method = registry.get(method_key or manifest.get("research_method"))
|
||||
if method_key:
|
||||
manifest["research_method"] = method.key
|
||||
titles = CHAPTER_TEMPLATES.get(method.key) or CHAPTER_TEMPLATES["mckinsey_market"]
|
||||
chapter_count = max(8, min(15, chapter_count))
|
||||
selected = titles[:chapter_count]
|
||||
quota = max(800, int(manifest.get("target_words", 30000)) // len(selected))
|
||||
sections = "\n".join(f"- {item}" for item in method.framework_sections)
|
||||
axes = "、".join(method.task_axes)
|
||||
material_text = render_material_inventory(manifest.get("material_inventory") or [])
|
||||
lines = [
|
||||
f"# {manifest.get('report_title') or manifest['topic']}:研究框架",
|
||||
"",
|
||||
f"research_method: {method.key}",
|
||||
f"method_name: {method.name}",
|
||||
f"work_language: 中文主写作;检索关键词、证据摘录、source title、raw notes 可保留英文。",
|
||||
f"target_words: {manifest.get('target_words', 30000)}",
|
||||
"",
|
||||
"## 方法选择",
|
||||
"",
|
||||
f"本项目采用 `{method.key}`,因为其结构原则是:{method.structure_principle}",
|
||||
"",
|
||||
"框架模块:",
|
||||
sections,
|
||||
"",
|
||||
"Phase 2 任务轴:",
|
||||
f"- {axes}",
|
||||
"",
|
||||
"## 输入材料与使用边界",
|
||||
"",
|
||||
material_text,
|
||||
"",
|
||||
"这些材料作为现场问题线索和内部事实起点使用;正式结论仍需结合 NMPA、FDA、EMA、ICH、WHO 等权威法规、指南和最佳实践进行验证。",
|
||||
"",
|
||||
"## 中心假设",
|
||||
"",
|
||||
f"围绕“{manifest['topic']}”形成可被证据支持或证伪的中文主线;所有核心判断必须绑定来源 ID。",
|
||||
"",
|
||||
]
|
||||
for idx, title in enumerate(selected, start=1):
|
||||
lines.extend(
|
||||
[
|
||||
f"## 第{idx}章 {title}",
|
||||
"",
|
||||
f"建议字数:约 {quota} 字。",
|
||||
f"研究思路:围绕 `{method.key}` 的方法框架,从 {axes} 等任务轴并发收集 evidence packet,再由 chapter assembly 收束为完整中文章节。",
|
||||
"证据要求:至少 2 个独立 Tier 1-2 信源;不足时在正文标注待验证;必须包含反方证据。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"## 暂停点",
|
||||
"",
|
||||
"请先确认 `phase1/material_brief.md` 的材料解读和访谈问题,再确认本框架后进入 Phase 2。若章节逻辑、方法框架或字数配额需要调整,应先修改本文件。",
|
||||
"",
|
||||
"确认后运行:`uv run python scripts/dr.py approve <project>`;未批准时 `research` 默认会拒绝推进,可用 `--force` 临时覆盖。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
out = project_root / "phase1" / "framework.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines), encoding="utf-8")
|
||||
manifest["phase1"] = {
|
||||
"status": "completed",
|
||||
"approved": False,
|
||||
"framework_path": "phase1/framework.md",
|
||||
"research_method": method.key,
|
||||
"updated_at": utc_now_iso(),
|
||||
}
|
||||
manifest["updated_at"] = utc_now_iso()
|
||||
write_manifest(project_root, manifest)
|
||||
return out
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Deterministic Phase 3 review checks for the Python core."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scripts.runtime.artifacts import load_manifest, write_manifest
|
||||
from scripts.runtime.tasks import validate_packet
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def _source_ids_from_jsonl(path: Path) -> set[str]:
|
||||
ids: set[str] = set()
|
||||
if not path.exists():
|
||||
return ids
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
source_id = obj.get("id") or obj.get("source_id")
|
||||
if source_id:
|
||||
ids.add(str(source_id))
|
||||
return ids
|
||||
|
||||
|
||||
def _draft_citations(drafts: list[Path]) -> set[str]:
|
||||
cited: set[str] = set()
|
||||
for draft in drafts:
|
||||
cited.update(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", draft.read_text(encoding="utf-8")))
|
||||
return cited
|
||||
|
||||
|
||||
def _ready_packet_stems(project_root: Path) -> set[str]:
|
||||
ready: set[str] = set()
|
||||
for path in sorted((project_root / "phase2" / "packets").glob("*.json")):
|
||||
try:
|
||||
packet = json.loads(path.read_text(encoding="utf-8"))
|
||||
validate_packet(packet)
|
||||
except Exception:
|
||||
continue
|
||||
ready.add(path.stem)
|
||||
return ready
|
||||
|
||||
|
||||
def _draft_quality_findings(drafts: list[Path]) -> list[dict[str, Any]]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
generic_markers = [
|
||||
"需要进一步完善",
|
||||
"应当加强",
|
||||
"持续改进",
|
||||
"系统性",
|
||||
"闭环管理",
|
||||
"质量文化",
|
||||
]
|
||||
for draft in drafts:
|
||||
text = draft.read_text(encoding="utf-8")
|
||||
zh_chars = sum(1 for char in text if "\u4e00" <= char <= "\u9fff")
|
||||
citations = re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", text)
|
||||
evidence_table_present = "证据落点" in text and "待补证据" in text
|
||||
if zh_chars >= 1200 and len(set(citations)) < 5:
|
||||
findings.append({"severity": "P1", "message": f"{draft.name} 引用来源过少,可能未充分使用 evidence packet。"})
|
||||
if zh_chars >= 1200 and not evidence_table_present:
|
||||
findings.append({"severity": "P1", "message": f"{draft.name} 缺少“证据落点与待补证据”小节,难以判断 evidence 是否真正落到纸面。"})
|
||||
generic_count = sum(text.count(marker) for marker in generic_markers)
|
||||
if zh_chars >= 1200 and generic_count >= 18:
|
||||
findings.append({"severity": "P1", "message": f"{draft.name} 泛化管理表述过多,需要回炉为具体审计发现、风险影响和整改动作。"})
|
||||
return findings
|
||||
|
||||
|
||||
def build_phase3_critique(project_root: Path) -> Path:
|
||||
manifest = load_manifest(project_root)
|
||||
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
|
||||
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
|
||||
ready_stems = _ready_packet_stems(project_root)
|
||||
packet_errors = [
|
||||
path for path in sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
|
||||
if path.stem not in ready_stems
|
||||
]
|
||||
chapter_errors = sorted((project_root / "phase2" / "chapter_errors").glob("*.json"))
|
||||
sources = _source_ids_from_jsonl(project_root / "phase2" / "sources.jsonl")
|
||||
cited = _draft_citations(drafts)
|
||||
missing_sources = sorted(cited - sources) if sources else sorted(cited)
|
||||
uncited_sources = sorted(sources - cited) if cited else sorted(sources)
|
||||
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not drafts:
|
||||
findings.append({"severity": "P1", "message": "Phase 2 drafts 缺失,尚不能进入 Phase 4 成稿。"})
|
||||
if packet_errors:
|
||||
findings.append({"severity": "P1", "message": f"存在 {len(packet_errors)} 个 packet 失败,需要回炉补证据。"})
|
||||
if chapter_errors:
|
||||
findings.append({"severity": "P1", "message": f"存在 {len(chapter_errors)} 个章节组装失败,需要修复引用或重写该章。"})
|
||||
quality_holds = manifest.get("quality_holds") or []
|
||||
if quality_holds:
|
||||
findings.append({"severity": "P1", "message": "存在质量暂停标记:" + ", ".join(quality_holds)})
|
||||
findings.extend(_draft_quality_findings(drafts))
|
||||
if missing_sources:
|
||||
findings.append({"severity": "P1", "message": f"正文引用未在 sources.jsonl 中登记:{', '.join(missing_sources)}"})
|
||||
if not findings:
|
||||
findings.append({"severity": "P2", "message": "基础产物完整;仍需人工或大上下文模型审校逻辑链、反方证据和章节叙事。"})
|
||||
|
||||
lines = [
|
||||
"# Phase 3 审校 critique",
|
||||
"",
|
||||
f"- 项目:{manifest.get('topic', project_root.name)}",
|
||||
f"- 运行时:python-core-v0.20",
|
||||
f"- drafts:{len(drafts)}",
|
||||
f"- packets:{len(packets)}",
|
||||
f"- sources:{len(sources)}",
|
||||
f"- cited_source_ids:{', '.join(sorted(cited)) if cited else '无'}",
|
||||
"",
|
||||
"## Findings",
|
||||
"",
|
||||
]
|
||||
for item in findings:
|
||||
lines.append(f"- [{item['severity']}] {item['message']}")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Residual Risks",
|
||||
"",
|
||||
"- 本 deterministic review 只做结构、引用和错误包检查;深层逻辑审校仍建议交给 `phase3_review` 角色执行。",
|
||||
"- 若 sources 为空,本审校会把所有正文引用视为待登记来源。",
|
||||
"",
|
||||
"## Next",
|
||||
"",
|
||||
"- 若存在 P1,先回到 Phase 2 修复 packet/chapter 错误。",
|
||||
"- 若仅有 P2,可进入 `dr.py finalize` 的中文原生成稿路径。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
out = project_root / "phase3" / "critique.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines), encoding="utf-8")
|
||||
phase3 = manifest.setdefault("phase3", {})
|
||||
phase3.update(
|
||||
{
|
||||
"status": "completed",
|
||||
"critique_path": "phase3/critique.md",
|
||||
"findings_total": len(findings),
|
||||
"missing_sources": missing_sources,
|
||||
"uncited_sources": uncited_sources,
|
||||
"updated_at": utc_now_iso(),
|
||||
}
|
||||
)
|
||||
manifest["updated_at"] = utc_now_iso()
|
||||
write_manifest(project_root, manifest)
|
||||
return out
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Runtime role and task-model resolution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from scripts.lib.model_config import resolve_model_profile
|
||||
|
||||
|
||||
ROLE_DEFAULTS = {
|
||||
"dr_plan": {
|
||||
"skills": ["document-ingest", "search-gateway", "search-strategy", "source-quality", "length-budget", "mckinsey-method"],
|
||||
"temperature": 0.4,
|
||||
"max_tokens": 12000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
"dr_pm": {
|
||||
"skills": ["length-budget", "evidence-table", "mckinsey-method"],
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 8000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
"dr_searcher": {
|
||||
"skills": ["search-gateway", "search-strategy", "source-quality"],
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 6000,
|
||||
"max_concurrency": 6,
|
||||
},
|
||||
"dr_analyst": {
|
||||
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table", "mckinsey-method"],
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 14000,
|
||||
"max_concurrency": 6,
|
||||
},
|
||||
"dr_verifier": {
|
||||
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table"],
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 10000,
|
||||
"max_concurrency": 4,
|
||||
},
|
||||
"dr_chief_editor": {
|
||||
"skills": ["mckinsey-method", "evidence-table", "output-hygiene"],
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 16000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
"dr_editor_in_chief": {
|
||||
"skills": ["mckinsey-method", "citation-manager", "humanizer-cn", "output-hygiene"],
|
||||
"temperature": 0.4,
|
||||
"max_tokens": 20000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
"dr_reporter": {
|
||||
"skills": ["pdf-reportlab", "citation-manager", "output-hygiene"],
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 6000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoleDefinition:
|
||||
name: str
|
||||
model: str
|
||||
skills: list[str]
|
||||
temperature: float
|
||||
max_tokens: int
|
||||
max_concurrency: int
|
||||
|
||||
|
||||
class RuntimeProfile:
|
||||
def __init__(self, *, profile: str, roles: dict[str, RoleDefinition], task_types: dict[str, str]) -> None:
|
||||
self.profile = profile
|
||||
self.roles = roles
|
||||
self.task_types = task_types
|
||||
|
||||
def role_for_task(self, task_type: str) -> RoleDefinition:
|
||||
role_name = self.task_types.get(task_type)
|
||||
if not role_name:
|
||||
raise KeyError(f"unknown task_type: {task_type}")
|
||||
if role_name not in self.roles:
|
||||
raise KeyError(f"task_type {task_type} maps to missing role {role_name}")
|
||||
return self.roles[role_name]
|
||||
|
||||
|
||||
def resolve_runtime_profile(
|
||||
*,
|
||||
profile: str | None = None,
|
||||
overrides: dict[str, str] | None = None,
|
||||
) -> RuntimeProfile:
|
||||
resolved = resolve_model_profile(profile=profile, overrides=overrides)
|
||||
role_models = resolved["roles"]
|
||||
roles: dict[str, RoleDefinition] = {}
|
||||
for name, defaults in ROLE_DEFAULTS.items():
|
||||
model = role_models.get(name)
|
||||
if not model:
|
||||
continue
|
||||
roles[name] = RoleDefinition(
|
||||
name=name,
|
||||
model=model,
|
||||
skills=list(defaults["skills"]),
|
||||
temperature=float(defaults["temperature"]),
|
||||
max_tokens=int(defaults["max_tokens"]),
|
||||
max_concurrency=int(defaults["max_concurrency"]),
|
||||
)
|
||||
return RuntimeProfile(
|
||||
profile=resolved["profile"],
|
||||
roles=roles,
|
||||
task_types=dict(resolved.get("task_types") or {}),
|
||||
)
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Canonical skill registry and adapter sync helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
CANONICAL_SKILLS_DIR = REPO_ROOT / ".agents" / "skills"
|
||||
PROJECT_SKILLS_DIR = REPO_ROOT / "skills"
|
||||
REQUIRED_SKILLS = {
|
||||
"search-strategy",
|
||||
"search-gateway",
|
||||
"source-quality",
|
||||
"length-budget",
|
||||
"evidence-table",
|
||||
"citation-manager",
|
||||
"mckinsey-method",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillInfo:
|
||||
name: str
|
||||
path: Path
|
||||
|
||||
|
||||
class SkillRegistry:
|
||||
"""Reads skills from the canonical cross-adapter registry."""
|
||||
|
||||
def __init__(self, canonical_dir: Path | None = None) -> None:
|
||||
self.canonical_dir = canonical_dir or CANONICAL_SKILLS_DIR
|
||||
|
||||
def roots(self) -> list[Path]:
|
||||
roots = [self.canonical_dir]
|
||||
if self.canonical_dir == CANONICAL_SKILLS_DIR and PROJECT_SKILLS_DIR.exists():
|
||||
roots.append(PROJECT_SKILLS_DIR)
|
||||
return roots
|
||||
|
||||
def list(self) -> list[SkillInfo]:
|
||||
seen: set[str] = set()
|
||||
out: list[SkillInfo] = []
|
||||
for root in self.roots():
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in sorted(root.glob("*/SKILL.md")):
|
||||
name = path.parent.name
|
||||
if name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
out.append(SkillInfo(name=name, path=path))
|
||||
return out
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
return [item.name for item in self.list()]
|
||||
|
||||
def read(self, name: str) -> str:
|
||||
for root in self.roots():
|
||||
path = root / name / "SKILL.md"
|
||||
if path.exists():
|
||||
return path.read_text(encoding="utf-8")
|
||||
raise FileNotFoundError(f"skill not found: {name}")
|
||||
|
||||
def validate(self, required: set[str] | None = None) -> dict[str, object]:
|
||||
names = set(self.list_names())
|
||||
required_names = required or REQUIRED_SKILLS
|
||||
missing = sorted(required_names - names)
|
||||
malformed: list[str] = []
|
||||
for item in self.list():
|
||||
text = item.path.read_text(encoding="utf-8")
|
||||
if "name:" not in text[:300]:
|
||||
malformed.append(item.name)
|
||||
return {
|
||||
"ok": not missing and not malformed,
|
||||
"canonical_dir": str(self.canonical_dir),
|
||||
"count": len(names),
|
||||
"missing": missing,
|
||||
"malformed": malformed,
|
||||
}
|
||||
|
||||
def sync_to(self, targets: list[Path], *, force: bool = True) -> int:
|
||||
"""Copy canonical skills into adapter skill directories.
|
||||
|
||||
Returns the number of skill directories copied across all targets.
|
||||
"""
|
||||
copied = 0
|
||||
for target in targets:
|
||||
if target.resolve() == self.canonical_dir.resolve():
|
||||
continue
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for item in self.list():
|
||||
dst = target / item.name
|
||||
if dst.exists() and force:
|
||||
shutil.rmtree(dst)
|
||||
if not dst.exists():
|
||||
shutil.copytree(item.path.parent, dst)
|
||||
copied += 1
|
||||
return copied
|
||||
|
||||
|
||||
def default_adapter_skill_dirs() -> list[Path]:
|
||||
return [
|
||||
REPO_ROOT / ".opencode" / "skills",
|
||||
REPO_ROOT / ".agents" / "skills",
|
||||
]
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Source registry helpers for Phase 2 packets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _source_key(source: dict[str, Any]) -> str:
|
||||
return (source.get("url") or source.get("doi") or source.get("id") or "").strip()
|
||||
|
||||
|
||||
def append_packet_sources(sources_path: Path, packet: dict[str, Any]) -> int:
|
||||
"""Append packet sources to sources.jsonl, deduping by URL/DOI/id."""
|
||||
sources_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
existing: set[str] = set()
|
||||
if sources_path.exists():
|
||||
for line in sources_path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
existing.add(_source_key(json.loads(line)))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
written = 0
|
||||
with sources_path.open("a", encoding="utf-8") as f:
|
||||
for source in packet.get("sources") or []:
|
||||
key = _source_key(source)
|
||||
if not key or key in existing:
|
||||
continue
|
||||
existing.add(key)
|
||||
f.write(json.dumps(source, ensure_ascii=False) + "\n")
|
||||
written += 1
|
||||
return written
|
||||
|
||||
|
||||
def rebuild_sources_from_packets(project_root: Path) -> int:
|
||||
"""Rebuild phase2/sources.jsonl from packet-level source metadata."""
|
||||
packets_dir = project_root / "phase2" / "packets"
|
||||
sources_path = project_root / "phase2" / "sources.jsonl"
|
||||
sources_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
seen: set[str] = set()
|
||||
rows: list[dict[str, Any]] = []
|
||||
|
||||
for packet_path in sorted(packets_dir.glob("*.json")):
|
||||
try:
|
||||
packet = json.loads(packet_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for source in packet.get("sources") or []:
|
||||
if not isinstance(source, dict):
|
||||
continue
|
||||
key = _source_key(source)
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
rows.append(source)
|
||||
|
||||
sources_path.write_text(
|
||||
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return len(rows)
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Task-card and evidence-packet primitives for v0.20 Phase 2."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scripts.runtime.methods import ResearchMethod
|
||||
|
||||
|
||||
VALID_ROUTES = {"general", "scholar", "patents", "news"}
|
||||
DEFAULT_AXES = ["literature", "regulatory", "patents", "market", "counter"]
|
||||
AXIS_ROUTES = {
|
||||
"literature": ["scholar", "general"],
|
||||
"clinical": ["scholar", "general"],
|
||||
"regulatory": ["general", "news"],
|
||||
"patents": ["patents", "general"],
|
||||
"market": ["news", "general"],
|
||||
"china": ["news", "general"],
|
||||
"counter": ["scholar", "general"],
|
||||
"regulatory_gap": ["general", "news"],
|
||||
"risk_classification": ["general", "scholar"],
|
||||
"capa_design": ["general", "news"],
|
||||
"ownership_timeline": ["general"],
|
||||
"verification_evidence": ["general", "scholar"],
|
||||
"process_flow": ["scholar", "general"],
|
||||
"cqa_cpp": ["scholar", "general"],
|
||||
"scale_up_risk": ["scholar", "general"],
|
||||
"control_strategy": ["scholar", "general"],
|
||||
"supply_chain": ["news", "general"],
|
||||
"scientific_rationale": ["scholar", "general"],
|
||||
"poc_evidence": ["scholar", "general"],
|
||||
"ip_fto": ["patents", "general"],
|
||||
"development_path": ["scholar", "general"],
|
||||
"commercial_window": ["news", "general"],
|
||||
"current_state": ["general"],
|
||||
"capability_gap": ["general"],
|
||||
"operating_model": ["general"],
|
||||
"governance": ["general"],
|
||||
"implementation_roadmap": ["general"],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chapter:
|
||||
chapter_id: str
|
||||
index: int
|
||||
title: str
|
||||
notes: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskCard:
|
||||
task_id: str
|
||||
chapter_ids: list[str]
|
||||
topic_axis: str
|
||||
questions: list[str]
|
||||
search_routes: list[str]
|
||||
output_packet: str
|
||||
preferred_model_role: str = "dr_analyst"
|
||||
status: str = "pending"
|
||||
dependencies: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def parse_framework_chapters(framework_text: str) -> list[Chapter]:
|
||||
"""Extract Chinese or English chapter headings from a framework markdown."""
|
||||
lines = framework_text.splitlines()
|
||||
chapters: list[Chapter] = []
|
||||
current: Chapter | None = None
|
||||
note_lines: list[str] = []
|
||||
heading_re = re.compile(
|
||||
r"^#{1,3}\s*(?:第\s*)?(\d{1,2})\s*(?:章|[.)、:-])?\s*(.+?)\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
english_re = re.compile(r"^#{1,3}\s*chapter\s+(\d{1,2})[:.)\s-]+(.+?)\s*$", re.IGNORECASE)
|
||||
for line in lines:
|
||||
match = heading_re.match(line.strip()) or english_re.match(line.strip())
|
||||
if match:
|
||||
if current:
|
||||
current.notes = "\n".join(note_lines).strip()
|
||||
chapters.append(current)
|
||||
index = int(match.group(1))
|
||||
title = match.group(2).strip(" #")
|
||||
current = Chapter(chapter_id=f"ch{index:02d}", index=index, title=title)
|
||||
note_lines = []
|
||||
elif current:
|
||||
note_lines.append(line)
|
||||
if current:
|
||||
current.notes = "\n".join(note_lines).strip()
|
||||
chapters.append(current)
|
||||
return chapters
|
||||
|
||||
|
||||
def _questions_for_axis(chapter: Chapter, axis: str) -> list[str]:
|
||||
return [
|
||||
f"围绕《{chapter.title}》从 {axis} 角度提炼可证伪的核心结论。",
|
||||
"至少寻找两个 Tier 1-2 来源支撑主要结论;不足时标注待验证。",
|
||||
"主动检索反方证据、限制条件或失败案例。",
|
||||
]
|
||||
|
||||
|
||||
def generate_task_cards(
|
||||
slug: str,
|
||||
framework_text: str,
|
||||
*,
|
||||
axes: list[str] | None = None,
|
||||
method: ResearchMethod | None = None,
|
||||
) -> list[TaskCard]:
|
||||
del slug # slug is kept for call-site clarity and future namespacing.
|
||||
chapters = parse_framework_chapters(framework_text)
|
||||
selected_axes = axes or (method.task_axes if method else DEFAULT_AXES)
|
||||
cards: list[TaskCard] = []
|
||||
for chapter in chapters:
|
||||
for axis in selected_axes:
|
||||
routes = AXIS_ROUTES.get(axis, ["general"])
|
||||
cards.append(
|
||||
TaskCard(
|
||||
task_id=f"{chapter.chapter_id}-{axis}",
|
||||
chapter_ids=[chapter.chapter_id],
|
||||
topic_axis=axis,
|
||||
questions=_questions_for_axis(chapter, axis),
|
||||
search_routes=routes,
|
||||
output_packet=f"phase2/packets/{chapter.chapter_id}-{axis}.json",
|
||||
preferred_model_role="dr_verifier" if axis == "counter" else "dr_analyst",
|
||||
)
|
||||
)
|
||||
validate_task_cards(cards)
|
||||
return cards
|
||||
|
||||
|
||||
def detect_dependency_cycles(cards: list[TaskCard]) -> None:
|
||||
graph = {card.task_id: card.dependencies for card in cards}
|
||||
visiting: set[str] = set()
|
||||
visited: set[str] = set()
|
||||
|
||||
def visit(node: str) -> None:
|
||||
if node in visiting:
|
||||
raise ValueError(f"dependency cycle detected at {node}")
|
||||
if node in visited:
|
||||
return
|
||||
visiting.add(node)
|
||||
for dep in graph.get(node, []):
|
||||
visit(dep)
|
||||
visiting.remove(node)
|
||||
visited.add(node)
|
||||
|
||||
for task_id in graph:
|
||||
visit(task_id)
|
||||
|
||||
|
||||
def validate_task_cards(cards: list[TaskCard]) -> None:
|
||||
seen: set[str] = set()
|
||||
for card in cards:
|
||||
if card.task_id in seen:
|
||||
raise ValueError(f"duplicate task_id: {card.task_id}")
|
||||
seen.add(card.task_id)
|
||||
if not card.chapter_ids:
|
||||
raise ValueError(f"{card.task_id}: chapter_ids required")
|
||||
if not card.questions:
|
||||
raise ValueError(f"{card.task_id}: questions required")
|
||||
if not card.output_packet.endswith(".json"):
|
||||
raise ValueError(f"{card.task_id}: output_packet must be json")
|
||||
invalid_routes = sorted(set(card.search_routes) - VALID_ROUTES)
|
||||
if invalid_routes:
|
||||
raise ValueError(f"{card.task_id}: invalid search_routes {invalid_routes}")
|
||||
missing_deps = sorted({dep for card in cards for dep in card.dependencies} - seen)
|
||||
if missing_deps:
|
||||
raise ValueError(f"unknown dependencies: {missing_deps}")
|
||||
detect_dependency_cycles(cards)
|
||||
|
||||
|
||||
def write_task_cards(path: Path, cards: list[TaskCard]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps([card.to_dict() for card in cards], ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def load_task_cards(path: Path) -> list[TaskCard]:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
cards = [TaskCard(**item) for item in data]
|
||||
validate_task_cards(cards)
|
||||
return cards
|
||||
|
||||
|
||||
def validate_packet(packet: dict[str, Any]) -> None:
|
||||
required = {
|
||||
"task_id",
|
||||
"claims",
|
||||
"evidence_items",
|
||||
"counter_evidence",
|
||||
"source_ids",
|
||||
"source_quality_notes",
|
||||
"open_questions",
|
||||
"raw_quotes_or_notes",
|
||||
}
|
||||
missing = sorted(required - set(packet))
|
||||
if missing:
|
||||
raise ValueError(f"packet missing fields: {missing}")
|
||||
if not packet["claims"]:
|
||||
raise ValueError("packet claims must not be empty")
|
||||
if not packet["evidence_items"]:
|
||||
raise ValueError("packet evidence_items must not be empty")
|
||||
if not packet["counter_evidence"]:
|
||||
raise ValueError("packet counter_evidence must not be empty")
|
||||
declared = set(packet.get("source_ids") or [])
|
||||
referenced: set[str] = set()
|
||||
for section in ("claims", "counter_evidence"):
|
||||
for item in packet.get(section) or []:
|
||||
referenced.update(item.get("source_ids") or [])
|
||||
for item in packet.get("evidence_items") or []:
|
||||
if item.get("source_id"):
|
||||
referenced.add(item["source_id"])
|
||||
undeclared = sorted(referenced - declared)
|
||||
if undeclared:
|
||||
raise ValueError(f"packet source_ids referenced but not declared: {undeclared}")
|
||||
packet_sources = packet.get("sources") or []
|
||||
if packet_sources:
|
||||
known_source_ids = {source.get("id") for source in packet_sources}
|
||||
missing_sources = sorted(declared - known_source_ids)
|
||||
if missing_sources:
|
||||
raise ValueError(f"packet source_ids missing source metadata: {missing_sources}")
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Python role workers for task-card execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Callable, Protocol, Any
|
||||
|
||||
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
|
||||
from scripts.runtime.skills import SkillRegistry
|
||||
from scripts.runtime.tasks import TaskCard, validate_packet
|
||||
from scripts.runtime.sources import append_packet_sources
|
||||
|
||||
|
||||
class ChatClient(Protocol):
|
||||
def chat_complete(self, **kwargs) -> str:
|
||||
...
|
||||
|
||||
|
||||
class SearchProvider(Protocol):
|
||||
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
|
||||
...
|
||||
|
||||
|
||||
class ProjectSearchProvider:
|
||||
"""Thin adapter over the project-owned search client."""
|
||||
|
||||
def __init__(self, *, strict_specialized: bool = True) -> None:
|
||||
from scripts.lib.search_client import SearchClient
|
||||
|
||||
self.client = SearchClient(strict_specialized=strict_specialized)
|
||||
|
||||
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
|
||||
if route == "scholar":
|
||||
hits = self.client.scholar(query, num_results=num_results, year_low=2020)
|
||||
elif route == "patents":
|
||||
hits = self.client.patents(query, num_results=num_results)
|
||||
elif route == "news":
|
||||
hits = self.client.news(query, num_results=num_results, time_range="y")
|
||||
else:
|
||||
hits = self.client.search(query, num_results=num_results)
|
||||
return [
|
||||
{
|
||||
"title": hit.title,
|
||||
"url": hit.url,
|
||||
"snippet": hit.snippet,
|
||||
"route": route,
|
||||
}
|
||||
for hit in hits
|
||||
]
|
||||
|
||||
def close(self) -> None:
|
||||
self.client.close()
|
||||
|
||||
|
||||
def _extract_json_object(text: str) -> dict:
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("```"):
|
||||
stripped = stripped.strip("`")
|
||||
if stripped.startswith("json"):
|
||||
stripped = stripped[4:].strip()
|
||||
start = stripped.find("{")
|
||||
end = stripped.rfind("}")
|
||||
if start == -1 or end == -1 or end < start:
|
||||
raise ValueError("worker response does not contain a JSON object")
|
||||
return json.loads(stripped[start : end + 1])
|
||||
|
||||
|
||||
def _safe_source_stem(task_id: str) -> str:
|
||||
return re.sub(r"[^a-zA-Z0-9]+", "_", task_id).strip("_").lower()
|
||||
|
||||
|
||||
def build_search_context(
|
||||
card: TaskCard,
|
||||
search_provider: SearchProvider,
|
||||
*,
|
||||
num_results_per_route: int = 5,
|
||||
) -> dict[str, Any]:
|
||||
candidate_sources: list[dict[str, Any]] = []
|
||||
routes_used: list[str] = []
|
||||
source_stem = _safe_source_stem(card.task_id)
|
||||
idx = 1
|
||||
query = " ".join(card.questions)
|
||||
for route in card.search_routes:
|
||||
routes_used.append(route)
|
||||
hits = search_provider.search(query=query, route=route, num_results=num_results_per_route)
|
||||
for hit in hits:
|
||||
candidate_sources.append(
|
||||
{
|
||||
"id": f"src_{source_stem}_{idx:03d}",
|
||||
"title": hit.get("title", ""),
|
||||
"url": hit.get("url", ""),
|
||||
"snippet": hit.get("snippet", ""),
|
||||
"route": hit.get("route", route),
|
||||
"tier": "Tier 2",
|
||||
"score": 6,
|
||||
}
|
||||
)
|
||||
idx += 1
|
||||
return {"routes_used": routes_used, "candidate_sources": candidate_sources}
|
||||
|
||||
|
||||
def build_packet_user_prompt(card: TaskCard, search_context: dict[str, Any] | None = None) -> str:
|
||||
context = search_context or {"routes_used": [], "candidate_sources": []}
|
||||
return (
|
||||
"请根据以下 task card 产出一个证据包 JSON。\n"
|
||||
"正式结论、summary、open_questions 用中文;英文原文摘录、source title、DOI/URL 可以保留英文。\n"
|
||||
"必须主动包含 counter_evidence,且所有引用的 source_id 必须出现在 source_ids 中。\n\n"
|
||||
"只能使用 candidate_sources 中的来源,不得编造 URL、DOI、trial ID 或 source_id。\n"
|
||||
"输出 JSON 必须包含 sources 字段,且 sources 只能来自 candidate_sources。\n\n"
|
||||
f"{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
|
||||
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
|
||||
"只输出 JSON,不要输出 Markdown 解释。"
|
||||
)
|
||||
|
||||
|
||||
def build_packet_repair_prompt(
|
||||
*,
|
||||
card: TaskCard,
|
||||
raw_response: str,
|
||||
error: Exception,
|
||||
search_context: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
context = search_context or {"routes_used": [], "candidate_sources": []}
|
||||
return (
|
||||
"请修复上一次 evidence packet 输出,使其成为合法且通过 schema 校验的 JSON。\n"
|
||||
"只输出 JSON 对象,不要输出 Markdown、解释或代码块。\n"
|
||||
"保留中文主写作;英文只允许出现在来源标题、URL、DOI、原文摘录或检索笔记中。\n"
|
||||
"不得编造 candidate_sources 以外的来源、URL、DOI、trial ID 或 source_id。\n\n"
|
||||
f"Schema error:\n{error}\n\n"
|
||||
f"Task card:\n{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
|
||||
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
|
||||
f"Previous raw response:\n{raw_response[:12000]}"
|
||||
)
|
||||
|
||||
|
||||
class PacketWorker:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
role: RoleDefinition,
|
||||
client: ChatClient,
|
||||
search_provider: SearchProvider | None = None,
|
||||
skill_registry: SkillRegistry | None = None,
|
||||
num_results_per_route: int = 5,
|
||||
) -> None:
|
||||
self.role = role
|
||||
self.client = client
|
||||
self.search_provider = search_provider
|
||||
self.skill_registry = skill_registry or SkillRegistry()
|
||||
self.num_results_per_route = num_results_per_route
|
||||
|
||||
def _system_prompt(self) -> str:
|
||||
skill_texts = []
|
||||
for name in self.role.skills:
|
||||
try:
|
||||
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
|
||||
except FileNotFoundError:
|
||||
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
|
||||
return (
|
||||
"你是 Deep Research v0.20 Python runtime 的证据包 worker。\n"
|
||||
"你的唯一任务是把一个 task card 转换为结构化 evidence packet。\n"
|
||||
"遵循中文主写作原则;不要写章节正文;不要编造 URL、DOI、trial ID 或 source_id。\n\n"
|
||||
"搜索只能走项目 Python search gateway 或调用方提供的 search_context;不要直接使用 Tavily MCP、browser MCP、平台 web search 或任何需要用户权限确认的外部搜索工具。\n\n"
|
||||
+ "\n\n".join(skill_texts)
|
||||
)
|
||||
|
||||
def run(self, card: TaskCard) -> dict:
|
||||
search_context = None
|
||||
if self.search_provider:
|
||||
search_context = build_search_context(
|
||||
card,
|
||||
self.search_provider,
|
||||
num_results_per_route=self.num_results_per_route,
|
||||
)
|
||||
raw = self.client.chat_complete(
|
||||
model=self.role.model,
|
||||
system=self._system_prompt(),
|
||||
user=build_packet_user_prompt(card, search_context),
|
||||
temperature=self.role.temperature,
|
||||
max_tokens=self.role.max_tokens,
|
||||
tag=f"packet:{card.task_id}",
|
||||
)
|
||||
try:
|
||||
packet = _extract_json_object(raw)
|
||||
validate_packet(packet)
|
||||
return packet
|
||||
except Exception as error:
|
||||
repaired = self.client.chat_complete(
|
||||
model=self.role.model,
|
||||
system=self._system_prompt(),
|
||||
user=build_packet_repair_prompt(
|
||||
card=card,
|
||||
raw_response=raw,
|
||||
error=error,
|
||||
search_context=search_context,
|
||||
),
|
||||
temperature=0,
|
||||
max_tokens=self.role.max_tokens,
|
||||
tag=f"packet-repair:{card.task_id}",
|
||||
)
|
||||
packet = _extract_json_object(repaired)
|
||||
validate_packet(packet)
|
||||
return packet
|
||||
|
||||
|
||||
def _write_packet_error(project_root: Path, card: TaskCard, error: Exception) -> None:
|
||||
path = project_root / "phase2" / "packet_errors" / f"{card.task_id}.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"task_id": card.task_id,
|
||||
"status": "failed",
|
||||
"error": str(error),
|
||||
"output_packet": card.output_packet,
|
||||
}
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def run_packet_workers(
|
||||
*,
|
||||
project_root: Path,
|
||||
cards: list[TaskCard],
|
||||
runtime: RuntimeProfile,
|
||||
client_factory: Callable[[RoleDefinition], ChatClient],
|
||||
search_provider_factory: Callable[[], SearchProvider] | None = None,
|
||||
workers: int,
|
||||
) -> int:
|
||||
role = runtime.role_for_task("evidence_packet")
|
||||
max_workers = max(1, min(workers, role.max_concurrency))
|
||||
|
||||
def run_one(card: TaskCard) -> tuple[TaskCard, dict | None, Exception | None]:
|
||||
search_provider = search_provider_factory() if search_provider_factory else None
|
||||
try:
|
||||
worker = PacketWorker(role=role, client=client_factory(role), search_provider=search_provider)
|
||||
return card, worker.run(card), None
|
||||
except Exception as error:
|
||||
return card, None, error
|
||||
finally:
|
||||
close = getattr(search_provider, "close", None)
|
||||
if close:
|
||||
close()
|
||||
|
||||
written = 0
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = [pool.submit(run_one, card) for card in cards]
|
||||
for future in as_completed(futures):
|
||||
card, packet, error = future.result()
|
||||
if error is not None:
|
||||
_write_packet_error(project_root, card, error)
|
||||
continue
|
||||
if packet is None:
|
||||
_write_packet_error(project_root, card, RuntimeError("packet worker returned no packet"))
|
||||
continue
|
||||
path = project_root / card.output_packet
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
append_packet_sources(project_root / "phase2" / "sources.jsonl", packet)
|
||||
written += 1
|
||||
return written
|
||||
Reference in New Issue
Block a user