feat(curator): vendor the application backend as the scenario's tracked source
The curator Python backend (package, tests, systemd units, config templates, scripts) now lives under scenarios/curator/backend and is the single source of truth; the live checkout at the workspace path is a runtime copy. Exported from the app repo's tracked tree via git archive (no history, .pi/venv/caches excluded). 149 unit tests pass from the new location. profile.toml backend is now repo-relative (scenarios/curator/backend); verify-generated.sh resolves a relative backend against REPO_ROOT. verify-no-secrets ASSIGN heuristic now requires value entropy so vendored kwargs like token=extraction_token no longer false-positive. README documents the backend/ layout and the operator-owned app rollout step.
This commit is contained in:
Kai
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from curator.epub import inspect_epub
|
||||
from curator.library import safe_component, safe_filename
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Merge one Curator book work into another")
|
||||
parser.add_argument("database", type=Path)
|
||||
parser.add_argument("target_work_id", type=int)
|
||||
parser.add_argument("source_work_id", type=int)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
connection = sqlite3.connect(args.database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
target = connection.execute("SELECT * FROM works WHERE id=?", (args.target_work_id,)).fetchone()
|
||||
source = connection.execute("SELECT * FROM works WHERE id=?", (args.source_work_id,)).fetchone()
|
||||
if not target or not source or target["media_type"] != "book" or source["media_type"] != "book":
|
||||
raise SystemExit("target and source must both be existing book works")
|
||||
rows = connection.execute(
|
||||
"""SELECT a.*, e.id AS source_edition_id, e.language, e.variant, e.isbn
|
||||
FROM assets a JOIN editions e ON e.id=a.edition_id WHERE e.work_id=?""",
|
||||
(args.source_work_id,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
raise SystemExit("source work has no assets")
|
||||
|
||||
changes: list[dict[str, object]] = []
|
||||
for row in rows:
|
||||
old_path = Path(row["path"])
|
||||
info = inspect_epub(old_path) if row["format"] == "epub" else None
|
||||
language = info.language if info else row["language"]
|
||||
edition_dir = safe_component(f"{language}-{row['variant']}", "und-original")
|
||||
destination_dir = old_path.parents[3] / safe_component(target["author"], "Unknown Author") / safe_component(target["title"], "Untitled") / edition_dir
|
||||
filename = safe_filename(row["filename"], f"book.{row['format']}")
|
||||
destination = destination_dir / filename
|
||||
metadata = json.loads(row["metadata_json"] or "{}")
|
||||
if info:
|
||||
metadata.update({
|
||||
"display_title": info.display_title,
|
||||
"title_aliases": list(info.title_aliases),
|
||||
"declared_language": info.declared_language,
|
||||
"identifiers": list(info.identifiers),
|
||||
"source_identifiers": list(info.source_identifiers),
|
||||
})
|
||||
changes.append({
|
||||
"asset_id": row["id"], "edition_id": row["source_edition_id"], "old": old_path,
|
||||
"new": destination, "filename": filename, "language": language,
|
||||
"isbn": info.isbn if info else row["isbn"], "metadata": metadata,
|
||||
})
|
||||
|
||||
print(json.dumps([{**change, "old": str(change["old"]), "new": str(change["new"])} for change in changes], ensure_ascii=False, indent=2, default=str))
|
||||
if not args.apply:
|
||||
return
|
||||
|
||||
backup = args.database.with_name(f"{args.database.stem}-before-merge-{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.sqlite3")
|
||||
with sqlite3.connect(backup) as backup_connection:
|
||||
connection.backup(backup_connection)
|
||||
moved: list[tuple[Path, Path]] = []
|
||||
try:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
for change in changes:
|
||||
old_path = change["old"]
|
||||
destination = change["new"]
|
||||
assert isinstance(old_path, Path) and isinstance(destination, Path)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if destination.exists() and destination != old_path:
|
||||
raise FileExistsError(destination)
|
||||
os.replace(old_path, destination)
|
||||
moved.append((destination, old_path))
|
||||
connection.execute(
|
||||
"UPDATE editions SET work_id=?, language=?, isbn=? WHERE id=?",
|
||||
(args.target_work_id, change["language"], change["isbn"], change["edition_id"]),
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE assets SET filename=?, path=?, metadata_json=? WHERE id=?",
|
||||
(change["filename"], str(destination), json.dumps(change["metadata"], ensure_ascii=False, sort_keys=True), change["asset_id"]),
|
||||
)
|
||||
connection.execute("DELETE FROM works WHERE id=?", (args.source_work_id,))
|
||||
connection.execute(
|
||||
"UPDATE works SET updated_at=? WHERE id=?",
|
||||
(datetime.now(UTC).replace(microsecond=0).isoformat(), args.target_work_id),
|
||||
)
|
||||
connection.commit()
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
for current, original in reversed(moved):
|
||||
original.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.replace(current, original)
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
print(f"backup={backup}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from curator.config import Settings
|
||||
from curator.db import Database, now
|
||||
|
||||
|
||||
LEGACY_PROVIDERS = {"google-books", "open-library"}
|
||||
|
||||
|
||||
def provider_is_legacy(value: object) -> bool:
|
||||
name = str(value or "").casefold()
|
||||
return any(name == provider or name.startswith(f"{provider}-") for provider in LEGACY_PROVIDERS)
|
||||
|
||||
|
||||
def scrub_metadata(metadata: dict[str, object]) -> bool:
|
||||
changed = False
|
||||
reviews = metadata.get("book_reviews")
|
||||
if isinstance(reviews, list):
|
||||
filtered = [item for item in reviews if not isinstance(item, dict) or not provider_is_legacy(item.get("provider"))]
|
||||
if filtered != reviews:
|
||||
metadata["book_reviews"] = filtered
|
||||
changed = True
|
||||
checked = metadata.get("book_review_providers_checked")
|
||||
if isinstance(checked, list):
|
||||
filtered = [item for item in checked if not provider_is_legacy(item)]
|
||||
if filtered != checked:
|
||||
metadata["book_review_providers_checked"] = filtered
|
||||
changed = True
|
||||
errors = metadata.get("book_review_errors")
|
||||
if isinstance(errors, list):
|
||||
filtered = [item for item in errors if not provider_is_legacy(str(item).split(":", 1)[0])]
|
||||
if filtered != errors:
|
||||
metadata["book_review_errors"] = filtered
|
||||
changed = True
|
||||
if changed:
|
||||
metadata["book_reviews_updated_at"] = ""
|
||||
return changed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Remove retired Google Books/Open Library data from Curator")
|
||||
parser.add_argument("--apply", action="store_true", help="write changes; otherwise report only")
|
||||
args = parser.parse_args()
|
||||
|
||||
settings = Settings.from_env()
|
||||
database = Database(settings.database)
|
||||
changed_rows: list[tuple[int, str]] = []
|
||||
with database.connect() as connection:
|
||||
for row in connection.execute("SELECT id,metadata_json FROM media_candidates"):
|
||||
metadata = json.loads(row["metadata_json"] or "{}")
|
||||
if scrub_metadata(metadata):
|
||||
changed_rows.append((int(row["id"]), json.dumps(metadata, ensure_ascii=False, sort_keys=True)))
|
||||
cache_count = int(connection.execute(
|
||||
"SELECT COUNT(*) FROM query_cache WHERE namespace='book-reviews'"
|
||||
).fetchone()[0])
|
||||
|
||||
legacy_covers: list[tuple[Path, list[Path]]] = []
|
||||
for sidecar in (settings.data_root / "covers").glob("*/*.json"):
|
||||
if sidecar.name.endswith(".missing.json"):
|
||||
continue
|
||||
try:
|
||||
metadata = json.loads(sidecar.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
if provider_is_legacy(metadata.get("provider")):
|
||||
images = list(sidecar.parent.glob(f"{sidecar.stem}.*"))
|
||||
legacy_covers.append((sidecar, [path for path in images if path != sidecar]))
|
||||
|
||||
result = {
|
||||
"apply": args.apply,
|
||||
"candidate_rows": len(changed_rows),
|
||||
"book_review_cache_rows": cache_count,
|
||||
"legacy_cover_sidecars": len(legacy_covers),
|
||||
}
|
||||
if not args.apply:
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
backup = settings.backup_root / "database" / "manual" / f"curator-before-book-provider-{stamp}.sqlite3"
|
||||
backup.parent.mkdir(parents=True, exist_ok=True)
|
||||
with sqlite3.connect(settings.database) as source, sqlite3.connect(backup) as destination:
|
||||
source.backup(destination)
|
||||
with database.connect() as connection:
|
||||
connection.executemany(
|
||||
"UPDATE media_candidates SET metadata_json=?,updated_at=? WHERE id=?",
|
||||
[(payload, now(), candidate_id) for candidate_id, payload in changed_rows],
|
||||
)
|
||||
connection.execute("DELETE FROM query_cache WHERE namespace='book-reviews'")
|
||||
for sidecar, images in legacy_covers:
|
||||
for image in images:
|
||||
image.unlink(missing_ok=True)
|
||||
sidecar.unlink(missing_ok=True)
|
||||
result["backup"] = str(backup)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from curator.db import Database, normalize
|
||||
from curator.epub import inspect_epub
|
||||
from curator.library import safe_component, safe_filename
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Repair a Curator EPUB from its embedded metadata")
|
||||
parser.add_argument("database", type=Path)
|
||||
parser.add_argument("asset_id", type=int)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
connection = sqlite3.connect(args.database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
row = connection.execute(
|
||||
"""SELECT a.*, e.id AS edition_id, e.work_id, e.variant
|
||||
FROM assets a JOIN editions e ON e.id=a.edition_id WHERE a.id=?""",
|
||||
(args.asset_id,),
|
||||
).fetchone()
|
||||
if not row or row["format"] != "epub":
|
||||
raise SystemExit("asset must be an existing EPUB")
|
||||
old_path = Path(row["path"])
|
||||
info = inspect_epub(old_path)
|
||||
filename = safe_filename(row["filename"], "book.epub")
|
||||
root = old_path.parents[3]
|
||||
destination = (
|
||||
root / safe_component(info.author, "Unknown Author") / safe_component(info.title, "Untitled")
|
||||
/ safe_component(f"{info.language}-{row['variant']}", "und-original") / filename
|
||||
)
|
||||
metadata = json.loads(row["metadata_json"] or "{}")
|
||||
metadata.update({
|
||||
"display_title": info.display_title,
|
||||
"title_aliases": list(info.title_aliases),
|
||||
"declared_language": info.declared_language,
|
||||
"identifiers": list(info.identifiers),
|
||||
"source_identifiers": list(info.source_identifiers),
|
||||
})
|
||||
plan = {"work_id": row["work_id"], "asset_id": row["id"], "title": info.title, "author": info.author,
|
||||
"language": info.language, "isbn": info.isbn, "old": str(old_path), "new": str(destination)}
|
||||
print(json.dumps(plan, ensure_ascii=False, indent=2))
|
||||
if not args.apply:
|
||||
return
|
||||
|
||||
backup = args.database.with_name(f"{args.database.stem}-before-repair-{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.sqlite3")
|
||||
with sqlite3.connect(backup) as backup_connection:
|
||||
connection.backup(backup_connection)
|
||||
moved = False
|
||||
try:
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if destination.exists() and destination != old_path:
|
||||
raise FileExistsError(destination)
|
||||
if destination != old_path:
|
||||
os.replace(old_path, destination)
|
||||
moved = True
|
||||
stamp = datetime.now(UTC).replace(microsecond=0).isoformat()
|
||||
connection.execute(
|
||||
"""UPDATE works SET title=?, author=?, normalized_title=?, normalized_author=?, updated_at=? WHERE id=?""",
|
||||
(info.title, info.author, normalize(info.title), normalize(info.author), stamp, row["work_id"]),
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE editions SET language=?, isbn=?, publisher=? WHERE id=?",
|
||||
(info.language, info.isbn, info.publisher, row["edition_id"]),
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE assets SET filename=?, path=?, metadata_json=? WHERE id=?",
|
||||
(filename, str(destination), json.dumps(metadata, ensure_ascii=False, sort_keys=True), row["id"]),
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE activity_jobs SET detail=?, updated_at=? WHERE kind='book-import' AND detail LIKE '%�%'",
|
||||
(f"{info.title} · {info.author or '未知作者'} · 已校验并入库", stamp),
|
||||
)
|
||||
connection.commit()
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
if moved:
|
||||
old_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.replace(destination, old_path)
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
Database(args.database).reconcile_imported_book(info.title, info.author)
|
||||
print(f"backup={backup}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user