release: v0.20 Codex-ready skill-driven core
This commit is contained in:
kai
@@ -0,0 +1,230 @@
|
||||
"""Cache important external sources as local Markdown snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from lxml import html
|
||||
|
||||
|
||||
IMPORTANT_DOMAINS = (
|
||||
"fda.gov",
|
||||
"ema.europa.eu",
|
||||
"nmpa.gov.cn",
|
||||
"cde.org.cn",
|
||||
"ich.org",
|
||||
"who.int",
|
||||
"edqm.eu",
|
||||
"pmda.go.jp",
|
||||
"ec.europa.eu",
|
||||
"health.ec.europa.eu",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CacheResult:
|
||||
source_id: str
|
||||
url: str
|
||||
cached_text_path: str
|
||||
raw_path: str
|
||||
status: str
|
||||
chars: int
|
||||
|
||||
|
||||
def _safe_stem(source: dict) -> str:
|
||||
source_id = str(source.get("id") or "source")
|
||||
digest = hashlib.sha1(str(source.get("url") or source_id).encode("utf-8")).hexdigest()[:10]
|
||||
safe_id = re.sub(r"[^A-Za-z0-9_-]+", "_", source_id).strip("_") or "source"
|
||||
return f"{safe_id}-{digest}"
|
||||
|
||||
|
||||
def _domain(url: str) -> str:
|
||||
return urlparse(url).netloc.lower()
|
||||
|
||||
|
||||
def is_important_source(source: dict) -> bool:
|
||||
url = str(source.get("url") or "")
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return False
|
||||
domain = _domain(url)
|
||||
if any(domain.endswith(item) for item in IMPORTANT_DOMAINS):
|
||||
return True
|
||||
tier = str(source.get("tier") or "").lower()
|
||||
if "tier 1" in tier or tier in {"1", "1.0"}:
|
||||
return True
|
||||
title = str(source.get("title") or "").lower()
|
||||
return any(term in title for term in ("ich q9", "ich q10", "annex 1", "fda guidance", "who guideline"))
|
||||
|
||||
|
||||
def load_sources(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows: list[dict] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
def write_sources(path: Path, rows: list[dict]) -> None:
|
||||
path.write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), encoding="utf-8")
|
||||
|
||||
|
||||
def _response_ext(url: str, content_type: str) -> str:
|
||||
lowered = url.lower()
|
||||
if "pdf" in content_type or lowered.endswith(".pdf"):
|
||||
return ".pdf"
|
||||
if "html" in content_type or lowered.endswith((".html", ".htm", "/")):
|
||||
return ".html"
|
||||
return ".bin"
|
||||
|
||||
|
||||
def _html_to_text(content: bytes) -> str:
|
||||
doc = html.fromstring(content)
|
||||
for bad in doc.xpath("//script|//style|//noscript"):
|
||||
bad.drop_tree()
|
||||
return "\n".join(line.strip() for line in doc.text_content().splitlines() if line.strip())
|
||||
|
||||
|
||||
def _pdf_to_text(path: Path) -> str:
|
||||
try:
|
||||
import fitz
|
||||
except Exception:
|
||||
return ""
|
||||
doc = fitz.open(path)
|
||||
parts: list[str] = []
|
||||
for index, page in enumerate(doc, start=1):
|
||||
text = page.get_text("text").strip()
|
||||
if text:
|
||||
parts.append(f"## Page {index}\n\n{text}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _bytes_to_text(*, raw_path: Path, content: bytes, content_type: str, url: str) -> str:
|
||||
if raw_path.suffix == ".pdf" or "pdf" in content_type or url.lower().endswith(".pdf"):
|
||||
return _pdf_to_text(raw_path)
|
||||
if raw_path.suffix in {".html", ".htm"} or "html" in content_type:
|
||||
return _html_to_text(content)
|
||||
try:
|
||||
return content.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return content.decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def cache_source(
|
||||
project_root: Path,
|
||||
source: dict,
|
||||
*,
|
||||
client: httpx.Client | None = None,
|
||||
force: bool = False,
|
||||
timeout: float = 45.0,
|
||||
) -> CacheResult:
|
||||
url = str(source.get("url") or "")
|
||||
if not url.startswith(("http://", "https://")):
|
||||
raise ValueError(f"source URL is not remote: {url}")
|
||||
cache_dir = project_root / "phase2" / "source_cache"
|
||||
raw_dir = cache_dir / "raw"
|
||||
text_dir = cache_dir / "md"
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
text_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
stem = _safe_stem(source)
|
||||
md_path = text_dir / f"{stem}.md"
|
||||
if md_path.exists() and not force:
|
||||
return CacheResult(
|
||||
source_id=str(source.get("id") or ""),
|
||||
url=url,
|
||||
cached_text_path=str(md_path.relative_to(project_root)),
|
||||
raw_path=str(source.get("cached_raw_path") or ""),
|
||||
status="cached",
|
||||
chars=len(md_path.read_text(encoding="utf-8")),
|
||||
)
|
||||
|
||||
owns_client = client is None
|
||||
http = client or httpx.Client(trust_env=False, follow_redirects=True, timeout=timeout)
|
||||
try:
|
||||
response = http.get(url)
|
||||
response.raise_for_status()
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
ext = _response_ext(str(response.url), content_type)
|
||||
raw_path = raw_dir / f"{stem}{ext}"
|
||||
raw_path.write_bytes(response.content)
|
||||
text = _bytes_to_text(raw_path=raw_path, content=response.content, content_type=content_type, url=str(response.url))
|
||||
lines = [
|
||||
f"# Source Snapshot: {source.get('title') or source.get('id') or url}",
|
||||
"",
|
||||
f"- source_id: {source.get('id', '')}",
|
||||
f"- original_url: {url}",
|
||||
f"- fetched_url: {response.url}",
|
||||
f"- content_type: {content_type}",
|
||||
f"- raw_path: {raw_path.relative_to(project_root)}",
|
||||
"",
|
||||
"## Extracted Text",
|
||||
"",
|
||||
text.strip() or "[No extractable text. Keep raw file for manual review.]",
|
||||
"",
|
||||
]
|
||||
md_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
return CacheResult(
|
||||
source_id=str(source.get("id") or ""),
|
||||
url=url,
|
||||
cached_text_path=str(md_path.relative_to(project_root)),
|
||||
raw_path=str(raw_path.relative_to(project_root)),
|
||||
status="fetched",
|
||||
chars=len(text),
|
||||
)
|
||||
finally:
|
||||
if owns_client:
|
||||
http.close()
|
||||
|
||||
|
||||
def cache_sources(
|
||||
project_root: Path,
|
||||
*,
|
||||
sources_rel: str = "phase2/sources.jsonl",
|
||||
important_only: bool = True,
|
||||
limit: int | None = None,
|
||||
force: bool = False,
|
||||
) -> list[CacheResult]:
|
||||
sources_path = project_root / sources_rel
|
||||
rows = load_sources(sources_path)
|
||||
results: list[CacheResult] = []
|
||||
selected_indexes = [
|
||||
index
|
||||
for index, row in enumerate(rows)
|
||||
if row.get("url")
|
||||
and (not row.get("cached_text_path") or force)
|
||||
and (not important_only or is_important_source(row))
|
||||
]
|
||||
if limit is not None:
|
||||
selected_indexes = selected_indexes[:limit]
|
||||
|
||||
with httpx.Client(trust_env=False, follow_redirects=True, timeout=45.0) as client:
|
||||
for index in selected_indexes:
|
||||
row = rows[index]
|
||||
try:
|
||||
result = cache_source(project_root, row, client=client, force=force)
|
||||
except Exception as exc:
|
||||
row["cache_status"] = "failed"
|
||||
row["cache_error"] = str(exc)[:300]
|
||||
continue
|
||||
row["cached_text_path"] = result.cached_text_path
|
||||
row["cached_raw_path"] = result.raw_path
|
||||
row["cache_status"] = result.status
|
||||
row["cached_text_chars"] = result.chars
|
||||
results.append(result)
|
||||
write_sources(sources_path, rows)
|
||||
manifest = project_root / "phase2" / "source_cache" / "manifest.json"
|
||||
manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest.write_text(
|
||||
json.dumps([result.__dict__ for result in results], ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return results
|
||||
Reference in New Issue
Block a user