runs/                              # JSONL 文件（source of truth）
└── 2026-05-27T...
    ├── meta.json
    └── samples.jsonl

.evalkit-dashboard/                # 本地 dashboard 存储（gitignored）
├── index.db                       # SQLite 索引
└── cache/

CREATE TABLE runs (
  id TEXT PRIMARY KEY,             -- run 目录名
  task_name TEXT,
  model TEXT,
  dataset_name TEXT,
  dataset_version TEXT,
  total_samples INTEGER,
  pass_rate REAL,
  total_cost_usd REAL,
  duration_ms INTEGER,
  started_at TEXT,
  completed_at TEXT,
  metadata JSON
);
 
CREATE TABLE sample_results (
  run_id TEXT,
  sample_id TEXT,
  passed INTEGER,                  -- boolean
  scores JSON,                     -- 多 scorer 结果
  failure_mode TEXT,                -- 如果挂，归类
  PRIMARY KEY (run_id, sample_id),
  FOREIGN KEY (run_id) REFERENCES runs(id)
);
 
CREATE INDEX idx_runs_task_started ON runs(task_name, started_at);
CREATE INDEX idx_samples_pass ON sample_results(run_id, passed);

// examples/evalkit/src/dashboard/sync.ts
export async function syncRunToDb(runDir: string, db: Database) {
  const meta = JSON.parse(readFileSync(`${runDir}/meta.json`, 'utf-8'));
  db.prepare(`
    INSERT OR REPLACE INTO runs (id, task_name, model, ...)
    VALUES (?, ?, ?, ...)
  `).run(meta.eval.taskName, ...);
 
  const samples = readFileSync(`${runDir}/samples.jsonl`, 'utf-8')
    .trim().split('\n')
    .map((l) => JSON.parse(l));
 
  const insertSample = db.prepare(`INSERT OR REPLACE INTO sample_results VALUES (?, ?, ?, ?, ?)`);
  for (const sample of samples) {
    insertSample.run(
      meta.id, sample.sampleId,
      sample.scores.every((s) => s.value === 'C') ? 1 : 0,
      JSON.stringify(sample.scores),
      classifyFailureMode(sample),
    );
  }
}

/                    Runs 列表（按时间 / task 排序）
/run/[id]            单 run 详情：metrics 卡片 + 200 条 samples 表格
/run/[id]/sample/[sid]  单 sample 详情：trace 完整渲染
/diff/[a]/[b]        两个 run 对比（improved / regressed 列表）
/trends/[task]       特定 task 的历史趋势：pass^1 曲线 / cost / failure mode 分布演化
/datasets            评测集列表 + 版本对比

// examples/evalkit/src/dashboard/app/page.tsx
export default async function RunsPage() {
  const runs = db.prepare(`
    SELECT * FROM runs
    ORDER BY started_at DESC
    LIMIT 50
  `).all();
 
  return (
    <div className="container mx-auto py-6">
      <h1 className="text-2xl font-semibold mb-6">Evaluation Runs</h1>
 
      <Table>
        <TableHeader>
          <TableRow>
            <TableHead>Started</TableHead>
            <TableHead>Task</TableHead>
            <TableHead>Model</TableHead>
            <TableHead>Samples</TableHead>
            <TableHead>Pass^1</TableHead>
            <TableHead>Cost</TableHead>
            <TableHead>Duration</TableHead>
          </TableRow>
        </TableHeader>
        <TableBody>
          {runs.map((run) => (
            <TableRow key={run.id}>
              <TableCell>{formatDate(run.started_at)}</TableCell>
              <TableCell>
                <Link href={`/run/${run.id}`} className="text-blue-600 hover:underline">
                  {run.task_name}
                </Link>
              </TableCell>
              <TableCell><Badge>{run.model}</Badge></TableCell>
              <TableCell>{run.total_samples}</TableCell>
              <TableCell>
                <PassRateBadge value={run.pass_rate} />
              </TableCell>
              <TableCell>${run.total_cost_usd.toFixed(2)}</TableCell>
              <TableCell>{(run.duration_ms / 1000).toFixed(0)}s</TableCell>
            </TableRow>
          ))}
        </TableBody>
      </Table>
    </div>
  );
}

// examples/evalkit/src/dashboard/app/trends/[task]/page.tsx
import { LineChart, Line, XAxis, YAxis, CartesianGrid, Tooltip, Legend } from 'recharts';
 
export default function TrendsPage({ params }: { params: { task: string } }) {
  const runs = db.prepare(`
    SELECT started_at, model, pass_rate, total_cost_usd
    FROM runs
    WHERE task_name = ? AND started_at > date('now', '-90 days')
    ORDER BY started_at
  `).all(params.task);
 
  // 按 model 分组（每个 model 一条线）
  const byModel = groupBy(runs, (r) => r.model);
 
  return (
    <div className="container mx-auto py-6">
      <h1>{params.task} Trend (last 90 days)</h1>
 
      <LineChart width={900} height={400} data={runs}>
        <CartesianGrid strokeDasharray="3 3" />
        <XAxis dataKey="started_at" tickFormatter={formatDate} />
        <YAxis domain={[0, 1]} tickFormatter={(v) => `${(v * 100).toFixed(0)}%`} />
        <Tooltip />
        <Legend />
        {Object.entries(byModel).map(([model, modelRuns]) => (
          <Line key={model} dataKey="pass_rate" data={modelRuns} stroke={modelColor(model)} name={model} />
        ))}
      </LineChart>
 
      <FailureModeEvolutionChart task={params.task} />
      <CostTrendChart task={params.task} />
    </div>
  );
}

/diff/<run-a-id>/<run-b-id>

// examples/evalkit/src/dashboard/app/diff/[a]/[b]/page.tsx
const { improved, regressed, stillFailing, unchanged } = diffRuns(runA, runB);
 
return (
  <div>
    <SummaryCard {...stats} />
 
    <Tabs defaultValue="regressed">
      <TabsList>
        <TabsTrigger value="regressed">Regressed ({regressed.length})</TabsTrigger>
        <TabsTrigger value="improved">Improved ({improved.length})</TabsTrigger>
        <TabsTrigger value="still">Still Failing ({stillFailing.length})</TabsTrigger>
      </TabsList>
      <TabsContent value="regressed">
        {regressed.map((id) => <SampleDiffRow sampleId={id} runA={runA} runB={runB} />)}
      </TabsContent>
      ...
    </Tabs>
  </div>
);

cd examples/evalkit
npm run dashboard

cd examples/evalkit/src/dashboard
npm run build
npm run start
# 或者 export 成静态 site:
# npm run export → out/ 部署到任意静态 hosting

git clone https://github.com/langfuse/langfuse
cd langfuse
docker-compose up -d

import { observeOpenAI } from 'langfuse';
 
const tracedClient = observeOpenAI(openaiClient);
// 所有 ShopAgent 的 LLM 调用自动 trace 到 langfuse

evalkit export-langfuse runs/<run-id> --base-url http://localhost:3000 --api-key xxx

import { Langfuse } from 'langfuse';
 
const lf = new Langfuse({ ... });
 
await lf.evaluation.create({
  name: 'shopagent-l1-v2',
  dataset: 'l1-v2.0.0',
  evaluator: (sample) => {
    // 你的 scorer，可以从 EvalKit 拷过来
  },
});

┌──────────────────────────────────────────────────┐
│ Production tracing (可选)                          │
│   - langfuse / Arize / 自研                        │
│   - 给数据飞轮提供 hard case 输入                    │
└──────────────────────────────────────────────────┘
                       ↓
┌──────────────────────────────────────────────────┐
│ Data Flywheel                                     │
│   - 4 类信号挖 hard case                           │
│   - LLM 预标 + 人工抽检                             │
│   - PII 清洗 + 入集                                │
└──────────────────────────────────────────────────┘
                       ↓
┌──────────────────────────────────────────────────┐
│ Eval Datasets (versioned)                         │
│   - L1 v2.x (200 条单轮)                           │
│   - L2 v2.x (100 条多轮)                           │
│   - L3 v1.x (40 条对抗)                            │
│   - RAG v1.x (50 条)                              │
└──────────────────────────────────────────────────┘
                       ↓
┌──────────────────────────────────────────────────┐
│ EvalKit Framework (2500 行 TS)                    │
│   - Task / Dataset / Solver / Scorer 四件套        │
│   - 11 个内置 scorer                               │
│   - Provider 路由 + 缓存 + 重试 + 并发              │
│   - JSONL EvalLog                                 │
│   - CLI: run / view / diff / list / ci            │
└──────────────────────────────────────────────────┘
                       ↓
┌──────────────────────────────────────────────────┐
│ Dashboard + CI                                    │
│   - Next.js Dashboard (本地 SQLite)                │
│   - GitHub Actions 守门                            │
│   - Slack/飞书通知                                  │
└──────────────────────────────────────────────────┘