┌──────────────────────────────────────────────┐
│ generation 层    用检索到的 context 生成回答      │
│   → faithfulness     回答是否忠于 context        │
│   → answer relevancy 回答是否答到用户问题         │
├──────────────────────────────────────────────┤
│ retrieval 层    把 query 转成向量，找 top-k 文档   │
│   → context precision  top-k 文档是否都相关       │
│   → context recall     该召回的文档是否都召回了     │
└──────────────────────────────────────────────┘

// examples/shopagent/src/tools/search_faq.ts —— 接口签名
export interface SearchFAQRequest {
  query: string;
  topK?: number;                 // 默认 3
}
 
export interface SearchFAQResult {
  hits: Array<{
    docId: string;
    title: string;
    content: string;             // 完整 FAQ 答案
    score: number;               // 0-1 相似度
  }>;
  generatedAnswer?: string;      // 如果 agent 要求生成答案，这里有
}

// examples/eval-datasets/rag/v1.0.0.jsonl
{
  "id": "RAG-001",
  "input": "羊毛衫能机洗吗？",
  "target": {
    "expectedContexts": ["手洗", "羊毛"],          // 该召回的 FAQ 关键词（用于 context_precision/recall）
    "expectedResponseContains": ["手洗", "羊毛"]  // 最终回复里必须出现的关键词（用于 includes scorer）
  },
  "metadata": {
    "category": "保养",
    "difficulty": "medium"
  }
}

precision = |relevant ∩ retrieved| / |retrieved|

// examples/evalkit/src/scorer/rag/context_precision.ts
// 实际是工厂函数返回 Scorer（跟其他 scorer 统一接口）
export function contextPrecision(opts: { field?: string; groundTruthField?: string } = {}): Scorer {
  return async (state, target) => {
    // 从 state.metadata.retrievedContexts 拿 retrieval 结果（solver 注入）
    // 从 target.expectedContexts 拿 ground truth 关键词
    // 算 precision = 相关数 / top-k
    // 返回 Score { value: 0-1, scorer: 'context_precision' }
    // 完整 80 行代码见仓库文件
  };
}

// examples/evalkit/src/scorer/rag/context_recall.ts
export function contextRecall(opts: { field?: string; groundTruthField?: string } = {}): Scorer {
  return async (state, target) => {
    // 从 target.expectedContexts 取 ground truth 关键词数组
    // 看其中多少个在 state.metadata.retrievedContexts 里出现
    // 返回 recalled / total_expected
  };
}

// examples/evalkit/src/scorer/rag/faithfulness.ts
const EXTRACT_CLAIMS_PROMPT = `把下面这段回答拆成独立的原子事实陈述，每行一条。
回答：{answer}
只输出陈述，不要其他文字。`;
 
const VERIFY_CLAIM_PROMPT = `判断下面的陈述是否被给定的 context 支持。
Context: {context}
Statement: {claim}
 
如果 context 明确支持该陈述，输出 1；如果 context 不包含该信息或与之矛盾，输出 0。
只输出一个数字。`;
 
export async function faithfulness(
  sample: { query: string },
  retrieved: Array<{ content: string }>,
  generated_answer: string,
  judge: Provider
): Promise<number> {
  const context = retrieved.map((d) => d.content).join('\n---\n');
 
  // 1. 提取断言
  const claimResp = await judge.generate({
    model: 'openai/gpt-4o-mini',  // judge 用便宜模型
    messages: [{ role: 'user', content: EXTRACT_CLAIMS_PROMPT.replace('{answer}', generated_answer) }],
    temperature: 0,
  });
  const claims = claimResp.content.trim().split('\n').filter((l) => l.trim().length > 0);
 
  if (claims.length === 0) return 0;
 
  // 2. 逐条判定
  let supported = 0;
  for (const claim of claims) {
    const verifyResp = await judge.generate({
      model: 'openai/gpt-4o-mini',
      messages: [{
        role: 'user',
        content: VERIFY_CLAIM_PROMPT
          .replace('{context}', context)
          .replace('{claim}', claim),
      }],
      temperature: 0,
      maxTokens: 5,
    });
    if (verifyResp.content.trim() === '1') supported++;
  }
 
  return supported / claims.length;
}

const verifyResults = await Promise.all(
  claims.map((claim) => judge.generate({ ... })),
);
const supported = verifyResults.filter((r) => r.content.trim() === '1').length;

const REVERSE_QUERY_PROMPT = `下面的回答可能是回答什么问题？给出 3 个可能的问题，每行一个。
回答：{answer}`;
 
export async function answerRelevancy(
  sample: { query: string },
  generated_answer: string,
  judge: Provider,
  embedder: (texts: string[]) => Promise<number[][]>
): Promise<number> {
  const resp = await judge.generate({
    model: 'openai/gpt-4o-mini',
    messages: [{ role: 'user', content: REVERSE_QUERY_PROMPT.replace('{answer}', generated_answer) }],
    temperature: 0,
  });
  const reverseQueries = resp.content.trim().split('\n').filter((l) => l.trim().length > 0);
 
  const allTexts = [sample.query, ...reverseQueries];
  const embs = await embedder(allTexts);
  const queryEmb = embs[0];
  const sims = embs.slice(1).map((e) => cosineSim(queryEmb, e));
 
  return sims.reduce((a, b) => a + b, 0) / sims.length;  // 平均相似度
}

// examples/ch08-rag/src/eval.ts
import { defineTask, jsonlDataset, getDefaultRouter } from '@inferloop/evalkit';
import { contextPrecision, contextRecall, faithfulness, answerRelevancy } from '@inferloop/evalkit/rag';
// 也可以从根路径 import { contextPrecision, ... } from '@inferloop/evalkit'
 
// ragSolver：自定义 solver，调 searchFaq 把结果塞到 state.metadata.retrievedContexts，再调 LLM 生成回复
import { ragSolver } from './rag_solver.js';
 
const router = getDefaultRouter();
const ragEval = defineTask({
  name: 'shopagent-faq-rag',
  dataset: jsonlDataset('../eval-datasets/rag/v1.0.0.jsonl', { limit: 50 }),
  solver: ragSolver({ topK: 3 }),
  scorer: [
    contextPrecision(),                                   // 工厂函数，返回 Scorer
    contextRecall(),
    faithfulness({ judgeRouter: router, judgeModel: 'gpt-4o' }),  // 需要 LLM judge
    answerRelevancy({ judgeRouter: router, judgeModel: 'gpt-4o' }),
  ],
  config: { temperature: 0 },
});

[evalkit] shopagent-faq-rag completed
  Context Precision  μ = 0.10   (top-3 里 10% 相关)
  Context Recall     μ = 0.10   (ground truth 10% 被召回)

Context Precision  μ ≈ 0.67
Context Recall     μ ≈ 0.81
Faithfulness       μ ≈ 0.92
Answer Relevancy   μ ≈ 0.85