HexTrails/scripts/bench-report.mjs

#!/usr/bin/env node
// Aggregate bench runs two ways: per level (is the level good?) and per model
// (how does this agent play?). Writes bench/report.json.
//   node scripts/bench-report.mjs [--dir=bench/runs] [--by=level|model|both] [--filter=prefix] [--json]
import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { metrics } from '../src/bench.js';

const flags = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
const dir = resolve(flags.dir || 'bench/runs');
if (!existsSync(dir)) { console.error(`没有 ${dir}`); process.exit(1); }
const runs = readdirSync(dir).filter(f => f.endsWith('.json') && f !== 'index.json')
  .filter(f => !flags.filter || f.startsWith(String(flags.filter)))
  .map(f => JSON.parse(readFileSync(resolve(dir, f), 'utf8')));
if (!runs.length) { console.error('没有符合条件的对局'); process.exit(1); }
const rows = runs.map(metrics);

const med = a => { const s = a.filter(v => typeof v === 'number').sort((x, y) => x - y); return s.length ? s[Math.floor(s.length / 2)] : null; };
const mean = a => { const s = a.filter(v => typeof v === 'number'); return s.length ? s.reduce((x, y) => x + y, 0) / s.length : null; };
const f = (v, d = 2) => v === null || v === undefined ? '—' : typeof v === 'number' ? Number(v.toFixed(d)) : v;
const secs = ms => ms == null ? '—' : ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`;
const ratingOf = ms => { const r = ms.map(m => m.rating).filter(Boolean); return Object.fromEntries(['clarity', 'rotation', 'stuck', 'fun', 'difficulty'].map(k => [k, mean(r.map(x => Number(x[k])))])); };

function group(keyOf) {
  const map = {};
  for (const m of rows) (map[keyOf(m)] ||= []).push(m);
  return Object.entries(map).map(([key, ms]) => {
    const finished = ms.filter(m => m.status !== 'playing');
    const cleared = ms.filter(m => m.cleared);
    return {
      key, runs: ms.length, finished: finished.length,
      clearRate: finished.length ? cleared.length / finished.length : null,
      gaveUp: ms.filter(m => m.gaveUp).length, budgetHit: ms.filter(m => m.status === 'budget').length,
      actionsPerPiece: med(cleared.map(m => m.actionsPerPiece)),
      decisions: med(ms.map(m => m.decisions)), actionsPerDecision: med(ms.map(m => m.actionsPerDecision)),
      totalMs: med(ms.map(m => m.timing.totalMs)), decisionMs: med(ms.map(m => m.timing.medianDecisionMs)),
      deadTaps: med(ms.map(m => m.deadTaps)), grazeDeadTaps: med(ms.map(m => m.grazeDeadTaps)),
      deadStreak: med(ms.map(m => m.longestDeadStreak)), blindStreak: med(ms.map(m => m.longestBlindStreak)),
      wrongWay: mean(ms.map(m => m.wrongWayRotations)), rotationEfficiency: mean(ms.map(m => m.rotationEfficiency)),
      extraRotations: med(cleared.filter(m => m.minimumRotations !== null).map(m => m.rotations - m.minimumRotations)),
      rating: ratingOf(ms), models: [...new Set(ms.map(m => m.model))], personas: [...new Set(ms.map(m => m.persona))], levels: [...new Set(ms.map(m => m.level))],
    };
  });
}

const byLevel = group(m => m.level);
byLevel.forEach(t => {
  const ms = rows.filter(m => m.level === t.key);
  t.flags = [t.gaveUp ? '有放弃' : null, ms.some(m => Number(m.rating?.stuck) >= 4) ? '有人觉得无解' : null,
    t.blindStreak >= 2 ? '盲转≥2' : null, t.budgetHit ? '预算用尽' : null].filter(Boolean);
});
byLevel.sort((a, b) => (b.flags.length - a.flags.length) || ((a.clearRate ?? 1) - (b.clearRate ?? 1)));
const byModel = group(m => `${m.model}${m.agent ? ` · ${m.agent}` : ''} · ${m.persona}`).sort((a, b) => (b.clearRate ?? 0) - (a.clearRate ?? 0) || (a.actionsPerPiece ?? 9) - (b.actionsPerPiece ?? 9));

const mode = flags.by || 'both';
if (flags.json) console.log(JSON.stringify({ runs: rows, levels: byLevel, models: byModel }, null, 1));
else {
  console.log(`bench 汇总 · ${runs.length} 局 · ${byLevel.length} 关 · ${byModel.length} 个 模型/人设 组合\n`);
  if (mode === 'model' || mode === 'both') {
    console.log('## 按模型（这个 agent 玩得怎么样）\n');
    console.log('| 模型 · 人设 | 局数 | 清盘率 | 放弃 | 动作÷棋子 | 决策数 | 步/决策 | 单局耗时 | 每决策 | 空点 | 擦碰 | 盲转 | 多转 | 清晰 | 旋转意义 | 无解感 | 好玩 | 难度 |');
    console.log('|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|');
    for (const t of byModel) console.log(`| ${t.key} | ${t.runs} | ${f(t.clearRate)} | ${t.gaveUp} | ${f(t.actionsPerPiece)} | ${f(t.decisions, 0)} | ${f(t.actionsPerDecision, 1)} | ${secs(t.totalMs)} | ${secs(t.decisionMs)} | ${f(t.deadTaps, 0)} | ${f(t.grazeDeadTaps, 0)} | ${f(t.blindStreak, 0)} | ${f(t.extraRotations, 0)} | ${f(t.rating.clarity, 1)} | ${f(t.rating.rotation, 1)} | ${f(t.rating.stuck, 1)} | ${f(t.rating.fun, 1)} | ${f(t.rating.difficulty, 1)} |`);
    console.log('');
  }
  if (mode === 'level' || mode === 'both') {
    console.log('## 按关卡（这一关好不好）\n');
    console.log('| 关卡 | 局数 | 清盘率 | 放弃 | 动作÷棋子 | 单局耗时 | 最长空点 | 最长盲转 | 多转 | 旋转效率 | 清晰 | 旋转意义 | 无解感 | 好玩 | 难度 | 标记 |');
    console.log('|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|');
    for (const t of byLevel) console.log(`| ${t.key} | ${t.runs} | ${f(t.clearRate)} | ${t.gaveUp} | ${f(t.actionsPerPiece)} | ${secs(t.totalMs)} | ${f(t.deadStreak, 0)} | ${f(t.blindStreak, 0)} | ${f(t.extraRotations, 0)} | ${f(t.rotationEfficiency)} | ${f(t.rating.clarity, 1)} | ${f(t.rating.rotation, 1)} | ${f(t.rating.stuck, 1)} | ${f(t.rating.fun, 1)} | ${f(t.rating.difficulty, 1)} | ${t.flags.join(' ')} |`);
    console.log('\n「无解感」= stuck 评分均值；「多转」= 清盘局的旋转次数 − 最少旋转数；「盲转」= 一次旋转什么都没打开；耗时含 agent 自己的进程启动。带标记的关先看。');
  }
}
writeFileSync(resolve('bench/report.json'), JSON.stringify({ generatedAt: new Date().toISOString(), runs: rows, levels: byLevel, models: byModel }, null, 1));