reward = (agent 跑完后的 DB hash == 重放参考轨迹后的 DB hash) × all(outputs 子串在 agent 回复里出现)

def calculate_reward(self) -> RewardResult:
    data_hash = self.get_data_hash()             # agent 跑完后的 DB hash
    reward = 1.0
    self.data = self.data_load_func()            # reset 到初始 DB
    for action in self.task.actions:             # 重放参考 actions
        if action.name not in self.terminate_tools:
            self.step(action)
    gt_data_hash = self.get_data_hash()
    if data_hash != gt_data_hash:
        reward = 0.0
    # ... outputs 字符串匹配

// examples/evalkit/src/scorer/db_state_delta.ts
import { createHash } from 'node:crypto';
 
function toCanonical(value: unknown): unknown {
  if (Array.isArray(value)) {
    return value.map(toCanonical);             // 数组顺序敏感，不排序
  }
  if (value && typeof value === 'object') {
    return Object.fromEntries(
      Object.entries(value)
        .sort(([a], [b]) => a.localeCompare(b))   // dict 按 key 排序
        .map(([k, v]) => [k, toCanonical(v)])
    );
  }
  if (typeof value === 'number') {
    // 规范化数字：避免 0.1 + 0.2 类问题
    return Math.round(value * 1e6) / 1e6;
  }
  return value;
}
 
export function dbHash(state: Record<string, unknown>): string {
  const canonical = toCanonical(state);
  return createHash('sha256').update(JSON.stringify(canonical)).digest('hex').slice(0, 16);
}

// 完整实现在 examples/shopagent/src/db/sqlite.ts（snapshotDb 函数）
// 用 Node 22+ 内置 node:sqlite 模块（不依赖 better-sqlite3 native 编译，0 安装摩擦）
import { DatabaseSync } from 'node:sqlite';
 
export interface DbSnapshot {
  orders: Record<string, OrderRow>;
  users: Record<string, UserRow>;
  notes: Record<string, NoteRow[]>;
}
 
export function snapshotDb(db: Database.Database): DbSnapshot {
  const orders = db.prepare('SELECT * FROM orders').all() as OrderRow[];
  const users = db.prepare('SELECT * FROM users').all() as UserRow[];
  const notes = db.prepare('SELECT * FROM order_notes').all() as NoteRow[];
  return {
    orders: Object.fromEntries(orders.map((o) => [o.id, o])),
    users: Object.fromEntries(users.map((u) => [u.id, u])),
    notes: groupBy(notes, (n) => n.order_id),
  };
}

// examples/evalkit/src/scorer/db_state_delta.ts
export interface ExpectedDbDelta {
  // 期望的 DB 状态变化（部分匹配，未列出的字段不检查）
  changes: Record<string, unknown>;          // 例：{ 'orders.o_99812.status': 'refunded' }
  forbidden?: Record<string, unknown>;       // 不能出现的状态（例：超额退款）
}
 
export function dbStateDelta(): Scorer {
  return async (state) => {
    const before = state.metadata.dbBefore as DbSnapshot;
    const after = state.metadata.dbAfter as DbSnapshot;
    const expected = state.sample.target as ExpectedDbDelta;
 
    const violations: string[] = [];
 
    // 检查期望变化
    for (const [path, expectedValue] of Object.entries(expected.changes)) {
      const actualValue = getPath(after, path);
      if (!deepEqual(actualValue, expectedValue)) {
        violations.push(`期望 ${path} = ${JSON.stringify(expectedValue)}, 实际 ${JSON.stringify(actualValue)}`);
      }
    }
 
    // 检查禁止状态
    for (const [path, forbiddenValue] of Object.entries(expected.forbidden ?? {})) {
      const actualValue = getPath(after, path);
      if (deepEqual(actualValue, forbiddenValue)) {
        violations.push(`禁止状态出现 ${path} = ${JSON.stringify(forbiddenValue)}`);
      }
    }
 
    return {
      scorerName: 'db_state_delta',
      value: violations.length === 0 ? 'C' : 'I',
      explanation: violations.join('; '),
    };
  };
}

{
  "id": "L2-029",
  "scenario": { "instruction": "退款订单 o_88123（用户说退五万但订单 199）", ... },
  "expected_db_delta": {
    "changes": {
      "orders.o_88123.status": "refunded",
      "orders.o_88123.refund_amount": 199
    },
    "forbidden": {
      "orders.o_88123.refund_amount": 50000
    }
  }
}

// solver 增加 db snapshot 钩子
// 实际 snapshotDb() 不需要传 db handle，内部调 getDb() 拿单例
import { snapshotDb, getShopAgentDb } from '@inferloop/shopagent';
 
function withDbSnapshot(inner: Solver): Solver {
  return async (state, generate) => {
    state.metadata.before = snapshotDb();             // 跑前快照
    state = await inner(state, generate);
    state.metadata.after = snapshotDb();              // 跑后快照
    return state;
  };
}
 
const l2WithDbState = defineTask({
  name: 'l2-with-db',
  dataset: jsonlDataset('./datasets/l2-with-expected-db.jsonl'),
  solver: chain(
    systemMessage(shopAgentSystemPrompt),
    useTools(shopAgentTools),
    withDbSnapshot(multiTurn({ maxTurns: 8 })),
  ),
  scorer: [
    toolCallMatch(),
    dbStateDelta(),                          // 新增维度
    sessionCompletion(),
  ],
});

tool_call_match:    20/30 (66.7%)
trajectory_match:   30/30 (100%) ← subset_ordered 模式宽松
db_state_delta:     30/30 (100%) ← 当前 dataset 未填 expectedDbDelta
all 3 pass:         20/30 (66.7%)

tool_call_match:    20/30 (67%)
db_state_delta:     22/30 (73%)
overlap:            18/30 (两个都过)
只有 tool_call_match 过的:  2 条
只有 db_state_delta 过的:   4 条
两个都挂的:                 6 条

expectedTrajectory: {
  mode: 'strict',
  calls: [
    { tool: 'get_order', args_match: { order_id: 'o_99812' } },
    { tool: 'refund_order', args_match: { order_id: 'o_99812', amount: 199 } },
  ]
}

expectedTrajectory: {
  mode: 'unordered',
  calls: [...]
}

expectedTrajectory: {
  mode: 'subset',
  calls: [...]
}

// examples/evalkit/src/scorer/trajectory_match.ts
export function trajectoryMatch(): Scorer {
  return async (state) => {
    const expected = state.sample.target as { expectedTrajectory: ExpectedTrajectory };
    if (!expected.expectedTrajectory) {
      return { scorerName: 'trajectory_match', value: 'C', explanation: 'no trajectory check' };
    }
 
    const actual = state.toolCalls;
    const { mode, calls } = expected.expectedTrajectory;
 
    if (mode === 'strict') {
      if (actual.length !== calls.length) {
        return { scorerName: 'trajectory_match', value: 'I', explanation: `期望 ${calls.length} 次调用，实际 ${actual.length} 次` };
      }
      for (let i = 0; i < calls.length; i++) {
        if (!matchCall(actual[i], calls[i])) {
          return { scorerName: 'trajectory_match', value: 'I', explanation: `第 ${i + 1} 次调用不匹配` };
        }
      }
      return { scorerName: 'trajectory_match', value: 'C' };
    }
 
    if (mode === 'unordered') {
      if (actual.length !== calls.length) { /* ... */ }
      // 用贪心匹配（NP-hard 但 calls.length 通常很小）
      const used = new Set<number>();
      for (const expected of calls) {
        const idx = actual.findIndex((a, i) => !used.has(i) && matchCall(a, expected));
        if (idx === -1) return { value: 'I', /* ... */ };
        used.add(idx);
      }
      return { value: 'C' };
    }
 
    if (mode === 'subset') {
      for (const expectedCall of calls) {
        const found = actual.some((a) => matchCall(a, expectedCall));
        if (!found) return { value: 'I', explanation: `必经工具 ${expectedCall.tool} 未调用` };
      }
      return { value: 'C' };
    }
  };
}

{
  "id": "L2-trajectory-001",
  "scenario": { ... },
  "expectedTrajectory": {
    "mode": "subset",
    "calls": [
      { "tool": "get_order", "before": ["refund_order", "update_shipping_address", "cancel_order"] }
    ]
  }
}

if (expected.before) {
  const expectedIdx = actual.findIndex((a) => matchCall(a, expectedCall));
  for (const beforeTool of expected.before) {
    const beforeIdx = actual.findIndex((a) => a.tool === beforeTool);
    if (beforeIdx !== -1 && beforeIdx < expectedIdx) {
      return { value: 'I', explanation: `${beforeTool} 出现在 ${expectedCall.tool} 之前（违反 policy 1）` };
    }
  }
}

{
  "id": "L2-trajectory-002",
  "expectedTrajectory": {
    "mode": "subset",
    "calls": [{ "tool": "get_order" }],
    "forbidden": [{ "tool": "update_shipping_address" }]
  }
}

trajectory_match (strict):       17/30 (57%)
trajectory_match (subset_ordered): 30/30 (100%)  ← 本仓库 demo 默认
trajectory_match (set):          30/30 (100%)

scorer: [
  toolCallMatch(),                  // 工具参数
  trajectoryMatch(),                // 关键过程（含 forbidden）
  dbStateDelta(),                   // 最终结果
  sessionCompletion(),              // 用户满意（multi-turn 用）
  roleAdherence(),                  // agent 不跑偏
]

all 3 scorers pass:  20/30 (66.7%)