66 lines
2.2 KiB
Python
66 lines
2.2 KiB
Python
"""Source registry helpers for Phase 2 packets."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def _source_key(source: dict[str, Any]) -> str:
|
|
return (source.get("url") or source.get("doi") or source.get("id") or "").strip()
|
|
|
|
|
|
def append_packet_sources(sources_path: Path, packet: dict[str, Any]) -> int:
|
|
"""Append packet sources to sources.jsonl, deduping by URL/DOI/id."""
|
|
sources_path.parent.mkdir(parents=True, exist_ok=True)
|
|
existing: set[str] = set()
|
|
if sources_path.exists():
|
|
for line in sources_path.read_text(encoding="utf-8").splitlines():
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
existing.add(_source_key(json.loads(line)))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
written = 0
|
|
with sources_path.open("a", encoding="utf-8") as f:
|
|
for source in packet.get("sources") or []:
|
|
key = _source_key(source)
|
|
if not key or key in existing:
|
|
continue
|
|
existing.add(key)
|
|
f.write(json.dumps(source, ensure_ascii=False) + "\n")
|
|
written += 1
|
|
return written
|
|
|
|
|
|
def rebuild_sources_from_packets(project_root: Path) -> int:
|
|
"""Rebuild phase2/sources.jsonl from packet-level source metadata."""
|
|
packets_dir = project_root / "phase2" / "packets"
|
|
sources_path = project_root / "phase2" / "sources.jsonl"
|
|
sources_path.parent.mkdir(parents=True, exist_ok=True)
|
|
seen: set[str] = set()
|
|
rows: list[dict[str, Any]] = []
|
|
|
|
for packet_path in sorted(packets_dir.glob("*.json")):
|
|
try:
|
|
packet = json.loads(packet_path.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
for source in packet.get("sources") or []:
|
|
if not isinstance(source, dict):
|
|
continue
|
|
key = _source_key(source)
|
|
if not key or key in seen:
|
|
continue
|
|
seen.add(key)
|
|
rows.append(source)
|
|
|
|
sources_path.write_text(
|
|
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
|
|
encoding="utf-8",
|
|
)
|
|
return len(rows)
|