HexTrails/scripts/bench.mjs

#!/usr/bin/env node
// Agent bench CLI. One command per step so any agent with a shell can play.
// Agents are given labs/bench/AGENT.md and need nothing else.
//   node scripts/bench.mjs list [--set=levels/prototypes/d5.json]
//   node scripts/bench.mjs start --model=NAME [--agent='运行环境 · 模式'] [--level=N] [--run=NAME] [--persona=…] [--set=…] [--peek] [--budget=80] [--no-rules]
//   node scripts/bench.mjs observe --run=NAME [--format=json|full]
//   node scripts/bench.mjs act --run=NAME tap t3, tap t5, rotate right   [--note=…] [--quiet]
//   node scripts/bench.mjs rate --run=NAME --json='{"clarity":3,…}'
//   node scripts/bench.mjs status|metrics --run=NAME [--json]
//   node scripts/bench.mjs open [--run=NAME]      # 起服务、打印/打开观察页
//   node scripts/bench.mjs rules [--peek]
// Runs live in bench/runs/NAME.json; the observer page polls that file.
import { spawn, spawnSync } from 'node:child_process';
import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync } from 'node:fs';
import { createConnection } from 'node:net';
import { resolve } from 'node:path';
import { createRun, observe, actMany, rate, metrics, rulesText, RATING_QUESTIONS, describeAction, describeResult } from '../src/bench.js';

const RUNS = resolve('bench/runs');
const argv = process.argv.slice(2);
const command = argv[0];
const flags = {}, positional = [];
for (const a of argv.slice(1)) {
  if (a.startsWith('--')) { const [k, ...v] = a.slice(2).split('='); flags[k] = v.length ? v.join('=') : true; }
  else positional.push(a);
}
const fail = message => { console.error(message); process.exit(1); };
// text = routes only (default, what a person sees); full = also the per-cell rail table; json = machine.
const fmt = () => flags.format === 'json' ? 'json' : flags.format === 'full' || flags.full ? 'full' : 'text';
const loadSet = path => {
  const raw = JSON.parse(readFileSync(resolve(path), 'utf8'));
  return Array.isArray(raw) ? raw : Array.isArray(raw.levels) ? raw.levels : Object.values(raw).filter(l => l && l.cells);
};
const runPath = name => resolve(RUNS, `${name}.json`);
const PORT = Number(process.env.PORT || 4173);
const slug = s => String(s).replace(/[^\w.-]+/g, '-').replace(/^-+|-+$/g, '') || 'agent';
// Name the run so the agent never has to invent one: model-L3-tapper, then -2, -3…
function autoRunName(model, level, persona) {
  const base = `${slug(model)}-L${level}-${slug(persona)}`;
  if (!existsSync(runPath(base))) return base;
  for (let n = 2; n < 99; n++) if (!existsSync(runPath(`${base}-${n}`))) return `${base}-${n}`;
  return `${base}-${Date.now()}`;
}
const portOpen = () => new Promise(done => {
  const socket = createConnection({ port: PORT, host: '127.0.0.1' });
  const finish = ok => { socket.destroy(); done(ok); };
  socket.setTimeout(400);
  socket.on('connect', () => finish(true));
  socket.on('error', () => finish(false));
  socket.on('timeout', () => finish(false));
});
// Bring the observer page within reach: start the static server if it is not
// already up, then print (and on macOS open) the URL for this run. Returns
// null in a portable pack, which ships no page and no server — a URL there
// would point at whatever else happens to hold the port.
const OBSERVER = resolve('labs/bench/index.html'), SERVER = resolve('scripts/serve.mjs');
async function observerUrl(run, { open = false } = {}) {
  if (!existsSync(OBSERVER) || !existsSync(SERVER)) return null;
  const url = `http://localhost:${PORT}/labs/bench/index.html${run ? `?run=${encodeURIComponent(run)}&live=1` : ''}`;
  if (!(await portOpen())) {
    const server = spawn(process.execPath, [SERVER], { detached: true, stdio: 'ignore' });
    server.unref();
    for (let i = 0; i < 20 && !(await portOpen()); i++) await new Promise(r => setTimeout(r, 150));
  }
  if (open && process.platform === 'darwin') spawnSync('open', [url], { stdio: 'ignore' });
  return url;
}
const loadRun = () => {
  if (!flags.run) fail('需要 --run=NAME');
  if (!existsSync(runPath(flags.run))) fail(`没有这局：${flags.run}（先 start）`);
  return JSON.parse(readFileSync(runPath(flags.run), 'utf8'));
};
const saveRun = runObj => {
  mkdirSync(RUNS, { recursive: true });
  runObj.updatedAt = new Date().toISOString();
  writeFileSync(runPath(runObj.run), JSON.stringify(runObj));
  const indexPath = resolve(RUNS, 'index.json');
  const index = existsSync(indexPath) ? JSON.parse(readFileSync(indexPath, 'utf8')) : [];
  const entry = { run: runObj.run, level: runObj.level.id, persona: runObj.persona, model: runObj.model, peek: runObj.peek, status: runObj.status, steps: runObj.steps.length, updatedAt: runObj.updatedAt };
  const at = index.findIndex(e => e.run === runObj.run);
  if (at >= 0) index[at] = entry; else index.push(entry);
  writeFileSync(indexPath, JSON.stringify(index, null, 1));
};

if (command === 'list') {
  const set = flags.set || 'levels/prototypes/d5.json';
  const levels = loadSet(set);
  console.log(`${set} · ${levels.length} 关`);
  levels.forEach((l, i) => console.log(`  ${String(i + 1).padStart(3)}  ${l.id}  ${l.cells.length} 格 · ${l.tiles.length} 枚${l.prototype ? ` · 锁住 ${Math.round(l.prototype.stats.lockedShare * 100)}% · 最少 ${l.prototype.stats.rotations} 转` : ''}`));
} else if (command === 'rules') {
  console.log(rulesText(!!flags.peek));
} else if (command === 'start') {
  const levels = loadSet(flags.set || 'levels/prototypes/d5.json');
  const wanted = flags.level ?? 1;
  const level = levels.find(l => l.id === wanted) || levels[Number(wanted) - 1];
  if (!level) fail(`找不到关卡 ${wanted}（本组共 ${levels.length} 关，用 list 查看）`);
  const persona = flags.persona || 'tapper';
  const model = flags.model || process.env.BENCH_MODEL || 'unknown';
  if (model === 'unknown') console.warn('提示：没有 --model，本局记为 unknown；填上你的模型标识才能按模型比较。');
  const name = flags.run ? String(flags.run) : autoRunName(model, levels.indexOf(level) + 1, persona);
  if (!/^[\w.-]+$/.test(name)) fail('run 名只能用字母、数字、_ - .');
  if (existsSync(runPath(name)) && !flags.force) fail(`${name} 已存在；换个名字或加 --force`);
  const agent = flags.agent ? String(flags.agent) : '';
  const runObj = createRun({ level, run: name, persona, model, agent, peek: !!flags.peek, budget: Number(flags.budget || 80), startedAt: new Date().toISOString() });
  saveRun(runObj);
  const url = await observerUrl(name, { open: !!flags.open });
  console.log(`开始 ${runObj.run} · 关卡 ${level.id} · 人设 ${runObj.persona} · 模型 ${runObj.model}${runObj.agent ? ` · ${runObj.agent}` : ''} · 预算 ${runObj.budget} 步 · peek ${runObj.peek ? '开' : '关'}`);
  if (!runObj.agent) console.warn('提示：没有 --agent，观察页横幅只会显示模型名。填上运行环境和模式，例如 --agent="Claude Code · ultracode"。');
  if (url) console.log(`观察页（人类实时观看）：${url}`);
  console.log(`后续命令都用 --run=${runObj.run}`);
  console.log('');
  // --no-rules: an agent that already read AGENT.md skips the rules block.
  if (!flags['no-rules']) { console.log(rulesText(runObj.peek)); console.log(''); }
  console.log(observe(runObj, { format: fmt() }));
} else if (command === 'open') {
  const url = await observerUrl(flags.run ? String(flags.run) : null, { open: flags.open !== 'false' });
  console.log(url || '这个副本没有观察页（便携包）；把 runs/ 交回原仓库用 bench-import.mjs 导入后再看。');
} else if (command === 'observe') {
  const runObj = loadRun();
  const view = observe(runObj, { format: fmt() });
  console.log(typeof view === 'string' ? view : JSON.stringify(view, null, 1));
} else if (command === 'act') {
  const runObj = loadRun();
  // One action, or several separated by commas: `tap t4, tap t5, rotate right`.
  // A chain stops at the first tap that fails and after any turn.
  const parse = words => {
    const [kind, ...rest] = words;
    if (kind === 'tap' || kind === 'peek') return { type: kind, id: rest[0] };
    if (kind === 'rotate') return { type: 'rotate', direction: ['right', 'r', '1', '+1', '右'].includes(rest[0]) ? 1 : ['left', 'l', '-1', '左'].includes(rest[0]) ? -1 : 0 };
    if (kind === 'undo') return { type: 'undo' };
    if (kind === 'giveup') return { type: 'giveup', reason: rest.join(' ') };
    fail('act 需要：tap ID | rotate left|right | undo | peek ID | giveup 原因；多个动作用逗号隔开');
  };
  const groups = positional.join(' ').split(',').map(s => s.trim()).filter(Boolean).map(s => parse(s.split(/\s+/)));
  if (!groups.length) fail('act 需要一个动作');
  if (groups.length > 8) fail('一次最多连做 8 个动作');
  const { outcomes, stopped } = actMany(runObj, groups, { note: flags.note ? String(flags.note) : '' });
  saveRun(runObj);
  for (const out of outcomes) console.log(`${describeAction(out.step?.action || out.rejected?.action)} → ${out.feedback}`);
  if (stopped) console.log(`（${stopped}）`);
  if (!flags.quiet && runObj.status === 'playing') { console.log(''); console.log(observe(runObj, { format: fmt() === 'json' ? 'text' : fmt() })); }
  if (runObj.status !== 'playing') console.log(`\n本局结束：${runObj.status}。请用 rate 提交评分。`);
  if (flags.json) console.log(JSON.stringify({ outcomes, stopped }));
} else if (command === 'rate') {
  const runObj = loadRun();
  let answers;
  try { answers = JSON.parse(flags.json || '{}'); } catch { fail('--json 需要合法 JSON'); }
  const rating = rate(runObj, answers);
  saveRun(runObj);
  console.log('已记录评分：' + JSON.stringify(rating));
  const missing = RATING_QUESTIONS.filter(q => rating[q.key] === null).map(q => q.key);
  if (missing.length) console.log('未填：' + missing.join(', '));
  // Close the loop for the agent: the numbers it should report, and the link.
  const m = metrics(runObj);
  const secs = ms => ms == null ? '—' : ms >= 1000 ? `${(ms / 1000).toFixed(1)} s` : `${ms} ms`;
  console.log(`本局：${m.status} · ${m.steps} 步 · ${m.decisions} 次决策 · 空点 ${m.deadTaps}（擦碰 ${m.grazeDeadTaps}）· 旋转 ${m.rotations}/${m.minimumRotations ?? '?'} · 耗时 ${secs(m.timing.totalMs)}`);
  const replayUrl = await observerUrl(runObj.run);
  if (replayUrl) console.log(`回放：${replayUrl}`);
} else if (command === 'status') {
  const runObj = loadRun();
  console.log(`${runObj.run} · ${runObj.level.id} · ${runObj.status} · ${runObj.steps.length}/${runObj.budget} 步 · 人设 ${runObj.persona} · 模型 ${runObj.model}`);
  for (const s of runObj.steps) console.log(`  ${String(s.n).padStart(3)}  ${describeAction(s.action).padEnd(10)} → ${describeResult(s.result)}${s.note ? `   「${s.note}」` : ''}`);
} else if (command === 'metrics') {
  const runObj = loadRun();
  const m = metrics(runObj);
  if (flags.json) console.log(JSON.stringify(m, null, 1));
  else console.log(Object.entries(m).filter(([k]) => k !== 'rating').map(([k, v]) => `${k}: ${typeof v === 'number' ? Number(v.toFixed(2)) : v}`).join('\n') + (m.rating ? '\nrating: ' + JSON.stringify(m.rating) : ''));
} else if (command === 'runs') {
  const dir = existsSync(RUNS) ? readdirSync(RUNS).filter(f => f.endsWith('.json') && f !== 'index.json') : [];
  for (const f of dir) { const r = JSON.parse(readFileSync(resolve(RUNS, f), 'utf8')); console.log(`${r.run.padEnd(28)} ${r.level.id.padEnd(34)} ${r.status.padEnd(9)} ${String(r.steps.length).padStart(3)} 步  ${r.persona}/${r.model}`); }
} else {
  fail('用法：node scripts/bench.mjs list|start|act|observe|rate|status|metrics|runs|open|rules …\n完整说明（给 Agent 读的那一份）：labs/bench/AGENT.md');
}