feat(curator): vendor the application backend as the scenario's tracked source

The curator Python backend (package, tests, systemd units, config templates, scripts) now lives under scenarios/curator/backend and is the single source of truth; the live checkout at the workspace path is a runtime copy. Exported from the app repo's tracked tree via git archive (no history, .pi/venv/caches excluded). 149 unit tests pass from the new location. profile.toml backend is now repo-relative (scenarios/curator/backend); verify-generated.sh resolves a relative backend against REPO_ROOT. verify-no-secrets ASSIGN heuristic now requires value entropy so vendored kwargs like token=extraction_token no longer false-positive. README documents the backend/ layout and the operator-owned app rollout step.
2026-08-30 18:49:10 -07:00
parent 14c97d88cf
commit 88b06d782f
57 changed files with 14236 additions and 9 deletions
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+from datetime import UTC, datetime
+from pathlib import Path
+
+from curator.config import Settings
+from curator.db import Database, now
+
+
+LEGACY_PROVIDERS = {"google-books", "open-library"}
+
+
+def provider_is_legacy(value: object) -> bool:
+    name = str(value or "").casefold()
+    return any(name == provider or name.startswith(f"{provider}-") for provider in LEGACY_PROVIDERS)
+
+
+def scrub_metadata(metadata: dict[str, object]) -> bool:
+    changed = False
+    reviews = metadata.get("book_reviews")
+    if isinstance(reviews, list):
+        filtered = [item for item in reviews if not isinstance(item, dict) or not provider_is_legacy(item.get("provider"))]
+        if filtered != reviews:
+            metadata["book_reviews"] = filtered
+            changed = True
+    checked = metadata.get("book_review_providers_checked")
+    if isinstance(checked, list):
+        filtered = [item for item in checked if not provider_is_legacy(item)]
+        if filtered != checked:
+            metadata["book_review_providers_checked"] = filtered
+            changed = True
+    errors = metadata.get("book_review_errors")
+    if isinstance(errors, list):
+        filtered = [item for item in errors if not provider_is_legacy(str(item).split(":", 1)[0])]
+        if filtered != errors:
+            metadata["book_review_errors"] = filtered
+            changed = True
+    if changed:
+        metadata["book_reviews_updated_at"] = ""
+    return changed
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Remove retired Google Books/Open Library data from Curator")
+    parser.add_argument("--apply", action="store_true", help="write changes; otherwise report only")
+    args = parser.parse_args()
+
+    settings = Settings.from_env()
+    database = Database(settings.database)
+    changed_rows: list[tuple[int, str]] = []
+    with database.connect() as connection:
+        for row in connection.execute("SELECT id,metadata_json FROM media_candidates"):
+            metadata = json.loads(row["metadata_json"] or "{}")
+            if scrub_metadata(metadata):
+                changed_rows.append((int(row["id"]), json.dumps(metadata, ensure_ascii=False, sort_keys=True)))
+        cache_count = int(connection.execute(
+            "SELECT COUNT(*) FROM query_cache WHERE namespace='book-reviews'"
+        ).fetchone()[0])
+
+    legacy_covers: list[tuple[Path, list[Path]]] = []
+    for sidecar in (settings.data_root / "covers").glob("*/*.json"):
+        if sidecar.name.endswith(".missing.json"):
+            continue
+        try:
+            metadata = json.loads(sidecar.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        if provider_is_legacy(metadata.get("provider")):
+            images = list(sidecar.parent.glob(f"{sidecar.stem}.*"))
+            legacy_covers.append((sidecar, [path for path in images if path != sidecar]))
+
+    result = {
+        "apply": args.apply,
+        "candidate_rows": len(changed_rows),
+        "book_review_cache_rows": cache_count,
+        "legacy_cover_sidecars": len(legacy_covers),
+    }
+    if not args.apply:
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+        return 0
+
+    stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+    backup = settings.backup_root / "database" / "manual" / f"curator-before-book-provider-{stamp}.sqlite3"
+    backup.parent.mkdir(parents=True, exist_ok=True)
+    with sqlite3.connect(settings.database) as source, sqlite3.connect(backup) as destination:
+        source.backup(destination)
+    with database.connect() as connection:
+        connection.executemany(
+            "UPDATE media_candidates SET metadata_json=?,updated_at=? WHERE id=?",
+            [(payload, now(), candidate_id) for candidate_id, payload in changed_rows],
+        )
+        connection.execute("DELETE FROM query_cache WHERE namespace='book-reviews'")
+    for sidecar, images in legacy_covers:
+        for image in images:
+            image.unlink(missing_ok=True)
+        sidecar.unlink(missing_ok=True)
+    result["backup"] = str(backup)
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())