"""Cache important external sources as local Markdown snapshots."""

from __future__ import annotations

import hashlib
import json
import re
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlparse

import httpx
from lxml import html


IMPORTANT_DOMAINS = (
    "fda.gov",
    "ema.europa.eu",
    "nmpa.gov.cn",
    "cde.org.cn",
    "ich.org",
    "who.int",
    "edqm.eu",
    "pmda.go.jp",
    "ec.europa.eu",
    "health.ec.europa.eu",
)


@dataclass(frozen=True)
class CacheResult:
    source_id: str
    url: str
    cached_text_path: str
    raw_path: str
    status: str
    chars: int


def _safe_stem(source: dict) -> str:
    source_id = str(source.get("id") or "source")
    digest = hashlib.sha1(str(source.get("url") or source_id).encode("utf-8")).hexdigest()[:10]
    safe_id = re.sub(r"[^A-Za-z0-9_-]+", "_", source_id).strip("_") or "source"
    return f"{safe_id}-{digest}"


def _domain(url: str) -> str:
    return urlparse(url).netloc.lower()


def is_important_source(source: dict) -> bool:
    url = str(source.get("url") or "")
    if not url.startswith(("http://", "https://")):
        return False
    domain = _domain(url)
    if any(domain.endswith(item) for item in IMPORTANT_DOMAINS):
        return True
    tier = str(source.get("tier") or "").lower()
    if "tier 1" in tier or tier in {"1", "1.0"}:
        return True
    title = str(source.get("title") or "").lower()
    return any(term in title for term in ("ich q9", "ich q10", "annex 1", "fda guidance", "who guideline"))


def load_sources(path: Path) -> list[dict]:
    if not path.exists():
        return []
    rows: list[dict] = []
    for line in path.read_text(encoding="utf-8").splitlines():
        if not line.strip():
            continue
        rows.append(json.loads(line))
    return rows


def write_sources(path: Path, rows: list[dict]) -> None:
    path.write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), encoding="utf-8")


def _response_ext(url: str, content_type: str) -> str:
    lowered = url.lower()
    if "pdf" in content_type or lowered.endswith(".pdf"):
        return ".pdf"
    if "html" in content_type or lowered.endswith((".html", ".htm", "/")):
        return ".html"
    return ".bin"


def _html_to_text(content: bytes) -> str:
    doc = html.fromstring(content)
    for bad in doc.xpath("//script|//style|//noscript"):
        bad.drop_tree()
    return "\n".join(line.strip() for line in doc.text_content().splitlines() if line.strip())


def _pdf_to_text(path: Path) -> str:
    try:
        import fitz
    except Exception:
        return ""
    doc = fitz.open(path)
    parts: list[str] = []
    for index, page in enumerate(doc, start=1):
        text = page.get_text("text").strip()
        if text:
            parts.append(f"## Page {index}\n\n{text}")
    return "\n\n".join(parts)


def _bytes_to_text(*, raw_path: Path, content: bytes, content_type: str, url: str) -> str:
    if raw_path.suffix == ".pdf" or "pdf" in content_type or url.lower().endswith(".pdf"):
        return _pdf_to_text(raw_path)
    if raw_path.suffix in {".html", ".htm"} or "html" in content_type:
        return _html_to_text(content)
    try:
        return content.decode("utf-8")
    except UnicodeDecodeError:
        return content.decode("utf-8", errors="ignore")


def cache_source(
    project_root: Path,
    source: dict,
    *,
    client: httpx.Client | None = None,
    force: bool = False,
    timeout: float = 45.0,
) -> CacheResult:
    url = str(source.get("url") or "")
    if not url.startswith(("http://", "https://")):
        raise ValueError(f"source URL is not remote: {url}")
    cache_dir = project_root / "phase2" / "source_cache"
    raw_dir = cache_dir / "raw"
    text_dir = cache_dir / "md"
    raw_dir.mkdir(parents=True, exist_ok=True)
    text_dir.mkdir(parents=True, exist_ok=True)

    stem = _safe_stem(source)
    md_path = text_dir / f"{stem}.md"
    if md_path.exists() and not force:
        return CacheResult(
            source_id=str(source.get("id") or ""),
            url=url,
            cached_text_path=str(md_path.relative_to(project_root)),
            raw_path=str(source.get("cached_raw_path") or ""),
            status="cached",
            chars=len(md_path.read_text(encoding="utf-8")),
        )

    owns_client = client is None
    http = client or httpx.Client(trust_env=False, follow_redirects=True, timeout=timeout)
    try:
        response = http.get(url)
        response.raise_for_status()
        content_type = response.headers.get("content-type", "").lower()
        ext = _response_ext(str(response.url), content_type)
        raw_path = raw_dir / f"{stem}{ext}"
        raw_path.write_bytes(response.content)
        text = _bytes_to_text(raw_path=raw_path, content=response.content, content_type=content_type, url=str(response.url))
        lines = [
            f"# Source Snapshot: {source.get('title') or source.get('id') or url}",
            "",
            f"- source_id: {source.get('id', '')}",
            f"- original_url: {url}",
            f"- fetched_url: {response.url}",
            f"- content_type: {content_type}",
            f"- raw_path: {raw_path.relative_to(project_root)}",
            "",
            "## Extracted Text",
            "",
            text.strip() or "[No extractable text. Keep raw file for manual review.]",
            "",
        ]
        md_path.write_text("\n".join(lines), encoding="utf-8")
        return CacheResult(
            source_id=str(source.get("id") or ""),
            url=url,
            cached_text_path=str(md_path.relative_to(project_root)),
            raw_path=str(raw_path.relative_to(project_root)),
            status="fetched",
            chars=len(text),
        )
    finally:
        if owns_client:
            http.close()


def cache_sources(
    project_root: Path,
    *,
    sources_rel: str = "phase2/sources.jsonl",
    important_only: bool = True,
    limit: int | None = None,
    force: bool = False,
) -> list[CacheResult]:
    sources_path = project_root / sources_rel
    rows = load_sources(sources_path)
    results: list[CacheResult] = []
    selected_indexes = [
        index
        for index, row in enumerate(rows)
        if row.get("url")
        and (not row.get("cached_text_path") or force)
        and (not important_only or is_important_source(row))
    ]
    if limit is not None:
        selected_indexes = selected_indexes[:limit]

    with httpx.Client(trust_env=False, follow_redirects=True, timeout=45.0) as client:
        for index in selected_indexes:
            row = rows[index]
            try:
                result = cache_source(project_root, row, client=client, force=force)
            except Exception as exc:
                row["cache_status"] = "failed"
                row["cache_error"] = str(exc)[:300]
                continue
            row["cached_text_path"] = result.cached_text_path
            row["cached_raw_path"] = result.raw_path
            row["cache_status"] = result.status
            row["cached_text_chars"] = result.chars
            results.append(result)
    write_sources(sources_path, rows)
    manifest = project_root / "phase2" / "source_cache" / "manifest.json"
    manifest.parent.mkdir(parents=True, exist_ok=True)
    manifest.write_text(
        json.dumps([result.__dict__ for result in results], ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )
    return results