release: v0.20 Codex-ready skill-driven core

scripts/runtime/sources.py

@@ -8,11 +8,11 @@ from typing import Any
 
 
 def _source_key(source: dict[str, Any]) -> str:
-    return (source.get("url") or source.get("doi") or source.get("id") or "").strip()
+    return (source.get("id") or source.get("source_id") or source.get("doi") or source.get("url") or "").strip()
 
 
 def append_packet_sources(sources_path: Path, packet: dict[str, Any]) -> int:
-    """Append packet sources to sources.jsonl, deduping by URL/DOI/id."""
+    """Append packet sources to sources.jsonl, preserving every citeable source_id."""
     sources_path.parent.mkdir(parents=True, exist_ok=True)
     existing: set[str] = set()
     if sources_path.exists():
@@ -37,10 +37,27 @@ def append_packet_sources(sources_path: Path, packet: dict[str, Any]) -> int:
 
 
 def rebuild_sources_from_packets(project_root: Path) -> int:
-    """Rebuild phase2/sources.jsonl from packet-level source metadata."""
+    """Rebuild phase2/sources.jsonl from packet-level source metadata.
+
+    The registry is keyed by source_id, not URL. Two packet sources may point to
+    the same URL but have different source_ids already cited in drafts; dropping
+    either row would break citation traceability.
+    """
     packets_dir = project_root / "phase2" / "packets"
     sources_path = project_root / "phase2" / "sources.jsonl"
     sources_path.parent.mkdir(parents=True, exist_ok=True)
+    existing_by_key: dict[str, dict[str, Any]] = {}
+    if sources_path.exists():
+        for line in sources_path.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            key = _source_key(row)
+            if key:
+                existing_by_key[key] = row
     seen: set[str] = set()
     rows: list[dict[str, Any]] = []
 
@@ -56,7 +73,8 @@ def rebuild_sources_from_packets(project_root: Path) -> int:
             if not key or key in seen:
                 continue
             seen.add(key)
-            rows.append(source)
+            previous = existing_by_key.get(key, {})
+            rows.append({**source, **{k: v for k, v in previous.items() if k.startswith("cache") or k.startswith("cached_")}})
 
     sources_path.write_text(
         "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),