"""
Retrieval evaluation for Docent AI.

For each question in eval_questions.json, runs the full retrieval pipeline
(hybrid BM25 + vector search + reranker) and checks whether a chunk from one
of the expected source PDFs appears in the results.

Reports:
  - hit@k   : fraction of questions where an expected file was retrieved
  - MRR     : mean reciprocal rank of the first expected file

Usage (from the project root):
    py evaluate.py
"""
import json
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent / "backend"))

from ingest import load_vector_store          # noqa: E402
from rag import build_engine, RETRIEVE_K      # noqa: E402

EVAL_FILE = Path(__file__).parent / "eval_questions.json"


def retrieve_with_retry(engine, question: str, attempts: int = 3):
    """The free NVIDIA endpoints occasionally return transient 5xx errors."""
    for i in range(attempts):
        try:
            return engine._retrieve(question)
        except Exception as e:
            if i == attempts - 1:
                raise
            print(f"  (transient {e.__class__.__name__} — retrying in 3s)")
            time.sleep(3)


def main() -> None:
    cases = json.loads(EVAL_FILE.read_text(encoding="utf-8"))

    store = load_vector_store()
    if store is None:
        print("No FAISS index found — ingest some PDFs first.")
        sys.exit(1)

    engine = build_engine(store)

    hits = 0
    reciprocal_ranks = []
    print(f"Evaluating {len(cases)} questions (k={RETRIEVE_K})\n")

    for case in cases:
        question = case["question"]
        expected = set(case["expected_files"])

        docs = retrieve_with_retry(engine, question)
        retrieved = [d.metadata.get("source", "?") for d in docs]

        rank = next(
            (i + 1 for i, f in enumerate(retrieved) if f in expected),
            None,
        )
        if rank is not None:
            hits += 1
            reciprocal_ranks.append(1.0 / rank)
            print(f"  HIT @{rank}  {question}")
        else:
            reciprocal_ranks.append(0.0)
            print(f"  MISS      {question}")
            print(f"            expected : {sorted(expected)}")
            print(f"            retrieved: {retrieved}")

    n = len(cases)
    print("\n" + "=" * 60)
    print(f"hit@{RETRIEVE_K} : {hits}/{n}  ({hits / n:.0%})")
    print(f"MRR    : {sum(reciprocal_ranks) / n:.3f}")


if __name__ == "__main__":
    main()