原始 judge 输出: policy_4_confirmation = 67/100 (67%)

观察 PASS = 真实 PASS × TPR + 真实 FAIL × (1 - TNR)
         = 75% × 0.923 + 25% × 0.125
         = 69.2% + 3.1%
         = 72.3%

θ̂ = (p_obs + TNR - 1) / (TPR + TNR - 1)

p_obs = 0.67
TPR = 0.923
TNR = 0.875

θ̂ = (0.67 + 0.875 - 1) / (0.923 + 0.875 - 1)
   = 0.545 / 0.798
   = 0.683

// examples/evalkit/src/stats/judgy.ts
export interface JudgePerformance {
  TPR: number;
  TNR: number;
}
 
export interface CalibratedEstimate {
  rawObservedPassRate: number;
  calibratedTrueRate: number;
  ci95: [number, number];          // 95% 置信区间
  warning?: string;                  // 如果 TPR + TNR < 1
}
 
export function calibrate(
  observedPasses: number,
  totalSamples: number,
  judgePerf: JudgePerformance,
  nBootstrap = 20000,
): CalibratedEstimate {
  const pObs = observedPasses / totalSamples;
  const denom = judgePerf.TPR + judgePerf.TNR - 1;
 
  if (denom <= 0) {
    return {
      rawObservedPassRate: pObs,
      calibratedTrueRate: pObs,
      ci95: [0, 1],
      warning: 'Judge TPR + TNR ≤ 1，校正失效，原始观察值不可信',
    };
  }
 
  const thetaHat = clamp((pObs + judgePerf.TNR - 1) / denom, 0, 1);
 
  // Bootstrap 置信区间
  const ci = bootstrapCi(observedPasses, totalSamples, judgePerf, nBootstrap);
 
  return {
    rawObservedPassRate: pObs,
    calibratedTrueRate: thetaHat,
    ci95: ci,
  };
}
 
function bootstrapCi(
  observedPasses: number,
  totalSamples: number,
  judgePerf: JudgePerformance,
  n: number,
): [number, number] {
  const estimates: number[] = [];
 
  for (let i = 0; i < n; i++) {
    // 重采样观察 pass 数（二项分布）
    const resampledPasses = binomialSample(totalSamples, observedPasses / totalSamples);
    const resampledPObs = resampledPasses / totalSamples;
    const denom = judgePerf.TPR + judgePerf.TNR - 1;
    const theta = clamp((resampledPObs + judgePerf.TNR - 1) / denom, 0, 1);
    estimates.push(theta);
  }
 
  estimates.sort((a, b) => a - b);
  return [estimates[Math.floor(n * 0.025)], estimates[Math.floor(n * 0.975)]];
}
 
function binomialSample(n: number, p: number): number {
  // 简化版二项采样：对每个样本独立 bernoulli
  let count = 0;
  for (let i = 0; i < n; i++) {
    if (Math.random() < p) count++;
  }
  return count;
}
 
function clamp(x: number, lo: number, hi: number): number {
  return Math.min(hi, Math.max(lo, x));
}

import { calibrate } from '@inferloop/evalkit/stats';
 
const result = calibrate(67, 100, { TPR: 0.923, TNR: 0.875 });
 
console.log(result);
// {
//   rawObservedPassRate: 0.67,
//   calibratedTrueRate: 0.683,
//   ci95: [0.58, 0.78]
// }

// examples/evalkit/src/scorer/judge/wrapper.ts
export function createJudge(opts: {
  prompt: string;
  performance: { TPR: number; TNR: number };
  measuredAt: string;
  measuredOn: string;  // dataset name
}): Judge {
  if (opts.performance.TPR < 0.8 || opts.performance.TNR < 0.8) {
    console.warn(`[judge] TPR=${opts.performance.TPR} TNR=${opts.performance.TNR}, below 80% threshold`);
  }
 
  const ageDays = (Date.now() - new Date(opts.measuredAt).getTime()) / (24 * 3600 * 1000);
  if (ageDays > 90) {
    console.warn(`[judge] TPR/TNR measured ${ageDays.toFixed(0)} days ago, please remeasure`);
  }
 
  return { /* ... */ };
}

// examples/evalkit/src/stats/bradley_terry.ts
export function bradleyTerry(
  wins: Array<{ winner: string; loser: string }>,
  agents: string[],
  iterations = 100,
): Record<string, number> {
  const ratings: Record<string, number> = Object.fromEntries(agents.map((a) => [a, 1]));
 
  for (let iter = 0; iter < iterations; iter++) {
    for (const a of agents) {
      let numerator = 0;
      let denominator = 0;
      for (const w of wins) {
        if (w.winner === a) {
          numerator += 1 / (ratings[a] + ratings[w.loser]);
        }
        if (w.loser === a) {
          denominator += 1 / (ratings[w.winner] + ratings[a]);
        }
      }
      // 防止零
      if (denominator > 0) ratings[a] = numerator / denominator;
    }
  }
 
  return ratings;
}

// examples/evalkit/src/stats/elo.ts
const K_FACTOR = 32;
 
export class EloRanking {
  private ratings = new Map<string, number>();
 
  constructor(initialRating = 1000) {
    this.initialRating = initialRating;
  }
 
  rate(agent: string): number {
    return this.ratings.get(agent) ?? this.initialRating;
  }
 
  recordMatch(winner: string, loser: string) {
    const wR = this.rate(winner);
    const lR = this.rate(loser);
    const expected = 1 / (1 + Math.pow(10, (lR - wR) / 400));
    this.ratings.set(winner, wR + K_FACTOR * (1 - expected));
    this.ratings.set(loser, lR + K_FACTOR * (0 - (1 - expected)));
  }
 
  ranking(): Array<{ agent: string; rating: number }> {
    return [...this.ratings.entries()]
      .map(([agent, rating]) => ({ agent, rating }))
      .sort((a, b) => b.rating - a.rating);
  }
}

async function pairwiseJudge(a: string, b: string, judge: Provider): Promise<Verdict> {
  const r1 = await judge.generate({ messages: [...buildPrompt(a, b)] });
  const r2 = await judge.generate({ messages: [...buildPrompt(b, a)] });
 
  if (r1.winner === 'A' && r2.winner === 'B') return 'tie';   // 都偏第一个
  if (r1.winner === 'A' && r2.winner === 'A') return 'a';     // 一致认为 a 好
  if (r1.winner === 'B' && r2.winner === 'B') return 'b';
  return 'tie';
}