# .github/workflows/evals.yml
name: ShopAgent Evaluation Gate
 
on:
  pull_request:
    branches: [main]
    paths:
      - 'examples/shopagent/**'      # ShopAgent 改动
      - 'examples/evalkit/**'         # 框架改动也跑（dogfood）
      - 'examples/eval-datasets/**'   # 评测集本身改动也跑（确认 baseline 不变）
 
jobs:
  l1-eval:
    runs-on: ubuntu-latest
    timeout-minutes: 30
 
    steps:
      - uses: actions/checkout@v4
 
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: npm
 
      - run: npm install
 
      - name: Doctor check
        run: npm run doctor
 
      - name: Run L1 eval (current branch)
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          MODEL: openai/gpt-4o
        run: |
          cd examples/ch17-dataset-v2
          EVALKIT_RUNS_DIR=runs MOCK_OUTPUT_NAME=current.jsonl npm run eval
 
      - name: Fetch baseline run from cache
        id: cache
        uses: actions/cache@v3
        with:
          path: runs/baseline-l1-v2.jsonl
          key: baseline-l1-v2-${{ hashFiles('examples/eval-datasets/l1/v2.0.0.jsonl') }}-${{ env.MODEL }}
 
      - name: Run L1 eval (main baseline) if cache miss
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          git checkout origin/main -- examples/shopagent
          cd examples/ch17-dataset-v2
          EVALKIT_RUNS_DIR=runs MOCK_OUTPUT_NAME=baseline-l1-v2.jsonl npm run eval
          git checkout HEAD -- examples/shopagent
 
      - name: Diff + statistical significance check
        id: gate
        run: |
          # evalkit ci 接收两个位置参数：baseline + candidate
          # 关键 flags：--accuracy-drop（默认 0.02）/ --p-value（默认 0.05）/ --regression-threshold（默认 0）
          npx evalkit ci \
            runs/baseline-l1-v2.jsonl \
            runs/current.jsonl \
            --accuracy-drop 0.03 \
            --p-value 0.05 \
            --regression-threshold 0 \
            > diff-report.md 2>&1 || echo "CI gate failed (see report)" >> diff-report.md
 
      - name: Post diff to PR
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const body = fs.readFileSync('diff-report.md', 'utf-8');
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body,
            });
 
      - name: Gate
        if: steps.gate.outputs.regression == 'true'
        run: |
          echo "::error::Significant regression detected"
          exit 1

baseline: p1 = 155/200 = 0.775, n1 = 200
current:  p2 = 150/200 = 0.750, n2 = 200

合并比例 p = (155 + 150) / 400 = 0.7625
SE = sqrt(p × (1-p) × (1/n1 + 1/n2))
   = sqrt(0.7625 × 0.2375 × 0.01)
   = sqrt(0.00181) = 0.0426

z = (0.775 - 0.750) / 0.0426 = 0.587
p-value (两侧) ≈ 0.56

// examples/evalkit/src/cli/ci.ts
import { twoProportionZTest } from '../stats/significance.js';
 
export interface CiOptions {
  baseline: string;          // baseline run jsonl
  current: string;            // current run jsonl
  threshold?: number;         // 最小可接受的 pass rate 差异，默认 0.03
  confidence?: number;        // 显著性阈值，默认 0.95
  output: string;             // diff report markdown 输出路径
}
 
export async function ciGate(opts: CiOptions): Promise<{ regression: boolean }> {
  const baseline = loadRun(opts.baseline);
  const current = loadRun(opts.current);
 
  const baselinePass = baseline.samples.filter(passEntirely).length;
  const currentPass = current.samples.filter(passEntirely).length;
 
  const z = twoProportionZTest(
    currentPass, current.samples.length,
    baselinePass, baseline.samples.length,
  );
 
  const alpha = 1 - (opts.confidence ?? 0.95);
  const significant = Math.abs(z.pValue) < alpha;
 
  const passRateDiff = (currentPass / current.samples.length) - (baselinePass / baseline.samples.length);
 
  // 三个判定：
  // 1. 显著下降 → regression
  // 2. 显著上升 → improvement
  // 3. 不显著 → noise
  let regression = false;
  let verdict: string;
  if (significant && passRateDiff < 0) {
    regression = true;
    verdict = `❌ Significant regression (Δ=${passRateDiff.toFixed(3)}, p=${z.pValue.toFixed(3)})`;
  } else if (significant && passRateDiff > 0) {
    verdict = `✅ Significant improvement (Δ=${passRateDiff.toFixed(3)}, p=${z.pValue.toFixed(3)})`;
  } else {
    verdict = `📊 No significant change (Δ=${passRateDiff.toFixed(3)}, p=${z.pValue.toFixed(3)})`;
  }
 
  // 写 diff 报告
  const report = generateDiffReport(baseline, current, verdict);
  writeFileSync(opts.output, report);
 
  return { regression };
}

## ShopAgent Evaluation Gate
 
### Verdict: ❌ Significant regression (Δ=-0.075, p=0.013)
 
| Metric | Baseline (main) | Current (PR) | Delta |
|---|---|---|---|
| pass^1 | 155/200 (77.5%) | 140/200 (70.0%) | -7.5pp |
| tool_call_match | 171/200 (85.5%) | 158/200 (79.0%) | -6.5pp |
| trajectory_subset | 164/200 (82.0%) | 161/200 (80.5%) | -1.5pp |
| policy_4_confirmation | 173/200 (86.5%) | 165/200 (82.5%) | -4.0pp |
| Cost | $0.18 | $0.17 | -$0.01 |
| Duration | 19s | 18s | -1s |
 
### Regressed Samples (15)
 
- L1-synth-021  was: ✓  →  now: 未调用期望工具 search_faq
- L1-synth-053  was: ✓  →  now: 工具 update_shipping_address 不应被调用
- L1-synth-013  was: ✓  →  now: 工具 refund_order 参数不匹配：amount
- ... (12 more)
 
### Improved Samples (3)
- L1-synth-040  was: ✗  →  now: ✓
- L1-synth-007  was: ✗  →  now: ✓
- L1-synth-049  was: ✗  →  now: ✓
 
### Failure Mode Distribution
 
| Failure Mode | Baseline | Current | Delta |
|---|---|---|---|
| FM-1 知识库未触发 | 4 | 8 | +4 ← 主要回归 |
| FM-2 退款金额诱导 | 1 | 1 | 0 |
| FM-3 已发货改地址 | 0 | 5 | +5 ← 主要回归 |
 
### Suggestion
 
主要回归集中在 FM-1 / FM-3。检查 PR #847 是否改动了：
- ShopAgent system prompt 中关于 search_faq 调用强制约束的措辞？
- update_shipping_address policy 1 / policy 2 的执行严格度？

每天 cost ≈ 5 × 0.9 × $0.25 + 5 × 0.1 × $5 = $1.13 + $2.5 = $3.6 / 天
每月 cost ≈ $110

# 快路径（PR 默认）
- L1 v2 单次（200 条）        $0.18, 19 秒
- L3 v1 单次（40 条）        $0.10, 8 秒
 
# 慢路径（PR 标 [full-eval] 标签触发）
- L1 v2 4 trials            $0.72, 1 分 16 秒
- L2 v2 单次（100 条）       $3.00, 3 分钟
- L3 v1 4 trials            $0.40, 32 秒

对 baseline pass^1 ≥ 90% 的样本：只跑 1 trial（这些样本稳定，不会在 pass^4 跌）
对 baseline pass^1 < 90% 的样本：跑 4 trial（暴露不稳定性）

strategy:
  matrix:
    model:
      - openai/gpt-4o          # always
      - openai/gpt-4o-mini      # if: contains(github.event.pull_request.labels.*.name, 'full-eval')
      - anthropic/claude-sonnet-4-5  # if: contains(github.event.pull_request.labels.*.name, 'full-eval')

class AdaptiveLimiter {
  private current = 5;
  private max = 50;
  private successInARow = 0;
 
  async run<T>(fn: () => Promise<T>): Promise<T> {
    while (!await this.tryAcquire()) await sleep(100);
    try {
      const result = await fn();
      this.onSuccess();
      return result;
    } catch (e) {
      if (this.is429(e)) this.onRateLimit();
      throw e;
    } finally {
      this.release();
    }
  }
 
  private onSuccess() {
    this.successInARow++;
    if (this.successInARow >= 10 && this.current < this.max) {
      this.current = Math.min(this.current * 1.5, this.max);
      this.successInARow = 0;
    }
  }
 
  private onRateLimit() {
    this.current = Math.max(1, Math.floor(this.current / 2));
    this.successInARow = 0;
  }
}

# 严格模式（默认）：任一显著回归 fail
evalkit ci --baseline x --current y --gate-mode strict
 
# 警告模式：评测集本身改动时用
evalkit ci --baseline x --current y --gate-mode warn
 
# 跳过模式：MAJOR 升级时用
evalkit ci --baseline x --current y --gate-mode skip

// .github/workflows/evals.yml 增加
- name: Notify regression
  if: steps.gate.outputs.regression == 'true'
  run: |
    curl -X POST $WEBHOOK_URL -d @- <<EOF
    {
      "msg_type": "text",
      "content": {
        "text": "ShopAgent eval regression on PR #${{ github.event.pull_request.number }}\n\nPR: ${{ github.event.pull_request.html_url }}\n\nVerdict: $(cat diff-report.md | head -1)"
      }
    }
    EOF

[CI Gate] ShopAgent eval regression on PR #1247

PR: https://github.com/diguike/book-agent-evals/pull/1247
Verdict: Significant regression (Δ=-0.075, p=0.013)

Metric:           Baseline      Current       Delta
pass^1            155/200       140/200       -7.5pp
tool_call_match   163/200       151/200       -6.0pp
trajectory_match  157/200       155/200       -1.0pp

Top 5 regressed samples:
- L1-127 (refund_happy_path)
- L1-082 (refund_happy_path)
- L1-145 (cancel_order)
...

- 任何商品相关知识问题必须先调 search_faq 工具查询
+ 商品相关问题应该优先调 search_faq，如果你确定知道答案也可以直接回答

Verdict: ❌ Significant regression (Δ=-0.045, p=0.018)

Regressed Samples (9):
- L1-synth-021  was: ✓  →  now: 未调用 search_faq
- L1-seed-006   was: ✓  →  now: 未调用 search_faq
- L1-synth-031  was: ✓  →  now: 未调用 search_faq
- ... 6 more