Why RAG needs its own evaluation

A RAG answer can be wrong for two completely different reasons — the retriever fetched the wrong context, or the generator mishandled good context. A single "is the answer right?" score can't tell them apart, and you can't fix what you can't localize. So RAG evaluation scores the two stages separately, then combines them. Recall the diagnostic reflex from RAG Fundamentals — check retrieval first — this topic turns that reflex into numbers.

The golden set — the foundation

Everything here depends on a golden set: a curated list of test cases, each with the question, the expected answer, and (crucially) the ids of the chunks that actually contain the answer. It's the ground truth you measure against.

from dataclasses import dataclass, field

@dataclass                                       # a typed record (auto-generated constructor)
class GoldenCase:
    question: str                                # the test question
    expected_answer: str                         # a reference answer (for answer-level scoring)
    relevant_chunk_ids: list[str] = field(default_factory=list)  # ids of chunks that DO contain the answer

golden = [
    GoldenCase("What does the Pro plan cost?", "€49/month", ["c12"]),
    GoldenCase("Is data encrypted at rest?",   "Yes, AES-256", ["c31", "c32"]),
    # …aim for 30–100 cases spanning easy, hard, and 'should abstain' questions…
]

Retriever metrics: context recall & precision

These score only retrieval, using the relevant_chunk_ids labels — no LLM needed, so they're cheap and deterministic.

• Context recall — of the chunks that should have been retrieved, how many were? Low recall = the answer wasn't even fetched → a retriever bug no prompt tuning can fix.
• Context precision — of the chunks that were retrieved, how many are actually relevant? Low precision = noise crowding the context window.

def context_recall(retrieved_ids: list[str], relevant_ids: list[str]) -> float:
    if not relevant_ids:                                    # guard: no ground-truth relevant chunks
        return 1.0
    hit = len(set(retrieved_ids) & set(relevant_ids))       # how many relevant chunks were retrieved (set intersection)
    return hit / len(relevant_ids)                          # fraction of the relevant set we recovered

def context_precision(retrieved_ids: list[str], relevant_ids: list[str]) -> float:
    if not retrieved_ids:                                   # guard: nothing retrieved
        return 0.0
    hit = len(set(retrieved_ids) & set(relevant_ids))       # how many retrieved chunks were actually relevant
    return hit / len(retrieved_ids)                         # fraction of what we returned that was on-target

# Example: retrieved 4 chunks, 1 of the 1 relevant chunk is among them:
context_recall(["c12", "c5", "c9", "c2"], ["c12"])    # -> 1.0  (found the answer chunk)
context_precision(["c12", "c5", "c9", "c2"], ["c12"]) # -> 0.25 (only 1 of 4 was relevant → noisy)

Generator metrics: faithfulness & answer relevance (LLM-as-judge)

Retrieval can be perfect and the answer still wrong. These metrics judge the generated answer, and since "is this answer grounded?" has no cheap formula, you use an LLM as the judge: a separate model call that scores the answer against a rubric.

• Faithfulness (groundedness) — is every claim in the answer supported by the retrieved context? Catches hallucination despite good retrieval.
• Answer relevance — does the answer actually address the question (not just say something true-but-useless)?

import json

def faithfulness(answer: str, context: str) -> float:
    # Ask a judge model to rule ONLY on whether the answer is supported by the context.
    prompt = (
        "You are a strict grader. Given CONTEXT and an ANSWER, decide whether every claim in the "
        "ANSWER is supported by the CONTEXT. Reply with JSON: {\"supported\": true|false, \"reason\": \"...\"}.\n\n"
        f"CONTEXT:\n{context}\n\nANSWER:\n{answer}"
    )
    raw = client.chat.completions.create(
        model="gpt-4o-mini",                               # a cheaper model is fine as a judge
        messages=[{"role": "user", "content": prompt}],
        temperature=0,                                     # deterministic grading — no creativity wanted
    ).choices[0].message.content
    verdict = json.loads(raw)                              # parse the JSON string into a dict (like JSON.parse)
    return 1.0 if verdict["supported"] else 0.0           # 1 = grounded, 0 = hallucinated

The regression harness

Individually the metrics are interesting; together, run over the golden set, they become a scorecard you can compare across changes. This is the artifact that lets you say "reranking raised faithfulness from 0.78 to 0.91" instead of "it feels better".

import statistics

def evaluate(pipeline, golden: list[GoldenCase]) -> dict:
    recalls, precisions, faiths = [], [], []              # collect per-case scores
    for case in golden:
        retrieved = pipeline.retrieve(case.question)       # -> list of chunks (each with an id + text)
        ids = [c.id for c in retrieved]
        context = "\n\n".join(c.text for c in retrieved)   # the context the generator will see
        answer = pipeline.answer(case.question)            # the generated answer

        recalls.append(context_recall(ids, case.relevant_chunk_ids))
        precisions.append(context_precision(ids, case.relevant_chunk_ids))
        faiths.append(faithfulness(answer, context))
    # Aggregate to a single scorecard (mean of each metric across all cases):
    return {
        "context_recall":    statistics.mean(recalls),
        "context_precision": statistics.mean(precisions),
        "faithfulness":      statistics.mean(faiths),
        "n": len(golden),
    }

# Compare two configurations on the SAME golden set — the only fair comparison:
base   = evaluate(baseline_pipeline, golden)    # e.g. {'context_recall': 0.72, 'faithfulness': 0.78, …}
reranked = evaluate(reranked_pipeline, golden)  # e.g. {'context_recall': 0.72, 'faithfulness': 0.91, …}
# Same recall, higher faithfulness → the rerank helped the GENERATOR, not retrieval. Now you know WHY.

Wire this into CI and a regression becomes a failing check, not a user complaint. Note how the paired scorecard localizes the effect — recall unchanged, faithfulness up — which is the entire point of scoring the stages separately.

Frameworks & online evaluation

You don't have to hand-roll all of this. Ragas and DeepEval package these exact metrics (context precision/recall, faithfulness, answer relevance) with reference implementations and LLM-judge prompts; Langfuse / Phoenix / LangSmith add tracing and let you run evals over real production traffic. Two modes:

• Offline (what we built) — run the golden set before shipping; a regression gate.
• Online — sample live traffic, log traces, and score a fraction with the same judges to catch drift the golden set never anticipated. (Observability is its own topic in the LLMOps track.)

Frameworks save boilerplate, but the discipline is identical: a golden set, stage-separated metrics, and a scorecard you compare across changes. Understand the metrics first; the library is just the runner.

Pitfalls

• No golden set. Without ground truth every "improvement" is a guess. Build it first.
• One blended score. A single accuracy number hides whether the failure is retrieval or generation — always score the stages separately.
• Trusting the judge blindly. LLM judges err; use temperature 0, a rubric, reasons, and periodic human spot-checks.
• Evaluating on the training/tuning data. Keep the golden set separate from anything you tuned on, or you measure memorization.
• Only happy-path cases. Omitting hard and abstain cases hides exactly the failures that hurt in production.

Recap

• RAG has two failure surfaces, so evaluation scores retriever and generator separately and combines them.
• The golden set (question → expected answer → relevant chunk ids) is the foundation — 30–100 cases including hard and abstain cases.
• Context recall/precision score retrieval cheaply from labels; faithfulness/answer relevance score generation via LLM-as-judge (temperature 0, rubric, reasons, spot-checks).
• A regression harness turns per-case metrics into a scorecard you compare across configs — and it localizes which stage a change affected.
• Ragas/DeepEval package the metrics and Langfuse/Phoenix add tracing + online eval; the discipline is the same either way.
• Worked examples map to the exercises: GoldenCase set (Ex 1), recall/precision (Ex 2), faithfulness judge (Ex 3), the evaluate harness (Ex 4).

from dataclasses import dataclass, field

@dataclass
class GoldenCase:
    question: str
    expected_answer: str
    relevant_chunk_ids: list = field(default_factory=list)

golden = [
    GoldenCase("What does the Pro plan cost?", "€49/month", ["c12"]),
    GoldenCase("Is data encrypted at rest?", "Yes, AES-256", ["c31"]),
    GoldenCase("What's the SLA uptime guarantee?", "99.9%", ["c40", "c41"]),  # split across two chunks
    GoldenCase("Do you support on-prem deployment?", "NOT_IN_CORPUS", []),     # abstain case
]

def context_recall(retrieved, relevant):
    if not relevant: return 1.0
    return len(set(retrieved) & set(relevant)) / len(relevant)

def context_precision(retrieved, relevant):
    if not retrieved: return 0.0
    return len(set(retrieved) & set(relevant)) / len(retrieved)

import statistics
rs, ps = [], []
for case in golden:
    ids = [c.id for c in retrieve(case.question, k=4)]
    rs.append(context_recall(ids, case.relevant_chunk_ids))
    ps.append(context_precision(ids, case.relevant_chunk_ids))
print(f"recall={statistics.mean(rs):.2f}  precision={statistics.mean(ps):.2f}")

import json
def faithfulness(answer, context):
    prompt = ("Strict grader. Is every claim in the ANSWER supported by the CONTEXT? "
              'Reply JSON: {"supported": true|false, "reason": "..."}.\n\n'
              f"CONTEXT:\n{context}\n\nANSWER:\n{answer}")
    raw = client.chat.completions.create(model="gpt-4o-mini", temperature=0,
        messages=[{"role":"user","content":prompt}]).choices[0].message.content
    v = json.loads(raw)
    return 1.0 if v["supported"] else 0.0

ctx = "The Pro plan costs €49/month and includes priority support."
print(faithfulness("The Pro plan is €49/month.", ctx))                       # 1.0
print(faithfulness("The Pro plan is €49/month and includes a free laptop.", ctx))  # 0.0 (invented)

import statistics
def evaluate(pipeline, golden):
    rs, ps, fs = [], [], []
    for case in golden:
        hits = pipeline.retrieve(case.question)
        ids = [c.id for c in hits]
        ctx = "\n\n".join(c.text for c in hits)
        ans = pipeline.answer(case.question)
        rs.append(context_recall(ids, case.relevant_chunk_ids))
        ps.append(context_precision(ids, case.relevant_chunk_ids))
        fs.append(faithfulness(ans, ctx))
    return {"recall": statistics.mean(rs), "precision": statistics.mean(ps),
            "faithfulness": statistics.mean(fs), "n": len(golden)}

print("baseline:", evaluate(baseline, golden))
print("reranked:", evaluate(reranked, golden))