Kai
88b06d782f
The curator Python backend (package, tests, systemd units, config templates, scripts) now lives under scenarios/curator/backend and is the single source of truth; the live checkout at the workspace path is a runtime copy. Exported from the app repo's tracked tree via git archive (no history, .pi/venv/caches excluded). 149 unit tests pass from the new location. profile.toml backend is now repo-relative (scenarios/curator/backend); verify-generated.sh resolves a relative backend against REPO_ROOT. verify-no-secrets ASSIGN heuristic now requires value entropy so vendored kwargs like token=extraction_token no longer false-positive. README documents the backend/ layout and the operator-owned app rollout step.
108 lines
4.2 KiB
Python
108 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sqlite3
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
from curator.config import Settings
|
|
from curator.db import Database, now
|
|
|
|
|
|
LEGACY_PROVIDERS = {"google-books", "open-library"}
|
|
|
|
|
|
def provider_is_legacy(value: object) -> bool:
|
|
name = str(value or "").casefold()
|
|
return any(name == provider or name.startswith(f"{provider}-") for provider in LEGACY_PROVIDERS)
|
|
|
|
|
|
def scrub_metadata(metadata: dict[str, object]) -> bool:
|
|
changed = False
|
|
reviews = metadata.get("book_reviews")
|
|
if isinstance(reviews, list):
|
|
filtered = [item for item in reviews if not isinstance(item, dict) or not provider_is_legacy(item.get("provider"))]
|
|
if filtered != reviews:
|
|
metadata["book_reviews"] = filtered
|
|
changed = True
|
|
checked = metadata.get("book_review_providers_checked")
|
|
if isinstance(checked, list):
|
|
filtered = [item for item in checked if not provider_is_legacy(item)]
|
|
if filtered != checked:
|
|
metadata["book_review_providers_checked"] = filtered
|
|
changed = True
|
|
errors = metadata.get("book_review_errors")
|
|
if isinstance(errors, list):
|
|
filtered = [item for item in errors if not provider_is_legacy(str(item).split(":", 1)[0])]
|
|
if filtered != errors:
|
|
metadata["book_review_errors"] = filtered
|
|
changed = True
|
|
if changed:
|
|
metadata["book_reviews_updated_at"] = ""
|
|
return changed
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Remove retired Google Books/Open Library data from Curator")
|
|
parser.add_argument("--apply", action="store_true", help="write changes; otherwise report only")
|
|
args = parser.parse_args()
|
|
|
|
settings = Settings.from_env()
|
|
database = Database(settings.database)
|
|
changed_rows: list[tuple[int, str]] = []
|
|
with database.connect() as connection:
|
|
for row in connection.execute("SELECT id,metadata_json FROM media_candidates"):
|
|
metadata = json.loads(row["metadata_json"] or "{}")
|
|
if scrub_metadata(metadata):
|
|
changed_rows.append((int(row["id"]), json.dumps(metadata, ensure_ascii=False, sort_keys=True)))
|
|
cache_count = int(connection.execute(
|
|
"SELECT COUNT(*) FROM query_cache WHERE namespace='book-reviews'"
|
|
).fetchone()[0])
|
|
|
|
legacy_covers: list[tuple[Path, list[Path]]] = []
|
|
for sidecar in (settings.data_root / "covers").glob("*/*.json"):
|
|
if sidecar.name.endswith(".missing.json"):
|
|
continue
|
|
try:
|
|
metadata = json.loads(sidecar.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
continue
|
|
if provider_is_legacy(metadata.get("provider")):
|
|
images = list(sidecar.parent.glob(f"{sidecar.stem}.*"))
|
|
legacy_covers.append((sidecar, [path for path in images if path != sidecar]))
|
|
|
|
result = {
|
|
"apply": args.apply,
|
|
"candidate_rows": len(changed_rows),
|
|
"book_review_cache_rows": cache_count,
|
|
"legacy_cover_sidecars": len(legacy_covers),
|
|
}
|
|
if not args.apply:
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
|
backup = settings.backup_root / "database" / "manual" / f"curator-before-book-provider-{stamp}.sqlite3"
|
|
backup.parent.mkdir(parents=True, exist_ok=True)
|
|
with sqlite3.connect(settings.database) as source, sqlite3.connect(backup) as destination:
|
|
source.backup(destination)
|
|
with database.connect() as connection:
|
|
connection.executemany(
|
|
"UPDATE media_candidates SET metadata_json=?,updated_at=? WHERE id=?",
|
|
[(payload, now(), candidate_id) for candidate_id, payload in changed_rows],
|
|
)
|
|
connection.execute("DELETE FROM query_cache WHERE namespace='book-reviews'")
|
|
for sidecar, images in legacy_covers:
|
|
for image in images:
|
|
image.unlink(missing_ok=True)
|
|
sidecar.unlink(missing_ok=True)
|
|
result["backup"] = str(backup)
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|