#!/usr/bin/env node // Agent bench CLI. One command per step so any agent with a shell can play. // Agents are given labs/bench/AGENT.md and need nothing else. // node scripts/bench.mjs list [--set=levels/prototypes/d5.json] // node scripts/bench.mjs start --model=NAME [--agent='运行环境 · 模式'] [--level=N] [--run=NAME] [--persona=…] [--set=…] [--peek] [--budget=80] [--no-rules] // node scripts/bench.mjs observe --run=NAME [--format=json|full] // node scripts/bench.mjs act --run=NAME tap t3, tap t5, rotate right [--note=…] [--quiet] // node scripts/bench.mjs rate --run=NAME --json='{"clarity":3,…}' // node scripts/bench.mjs status|metrics --run=NAME [--json] // node scripts/bench.mjs open [--run=NAME] # 起服务、打印/打开观察页 // node scripts/bench.mjs rules [--peek] // Runs live in bench/runs/NAME.json; the observer page polls that file. import { spawn, spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync } from 'node:fs'; import { createConnection } from 'node:net'; import { resolve } from 'node:path'; import { createRun, observe, actMany, rate, metrics, rulesText, RATING_QUESTIONS, describeAction, describeResult } from '../src/bench.js'; const RUNS = resolve('bench/runs'); const argv = process.argv.slice(2); const command = argv[0]; const flags = {}, positional = []; for (const a of argv.slice(1)) { if (a.startsWith('--')) { const [k, ...v] = a.slice(2).split('='); flags[k] = v.length ? v.join('=') : true; } else positional.push(a); } const fail = message => { console.error(message); process.exit(1); }; // text = routes only (default, what a person sees); full = also the per-cell rail table; json = machine. const fmt = () => flags.format === 'json' ? 'json' : flags.format === 'full' || flags.full ? 'full' : 'text'; const loadSet = path => { const raw = JSON.parse(readFileSync(resolve(path), 'utf8')); return Array.isArray(raw) ? raw : Array.isArray(raw.levels) ? raw.levels : Object.values(raw).filter(l => l && l.cells); }; const runPath = name => resolve(RUNS, `${name}.json`); const PORT = Number(process.env.PORT || 4173); const slug = s => String(s).replace(/[^\w.-]+/g, '-').replace(/^-+|-+$/g, '') || 'agent'; // Name the run so the agent never has to invent one: model-L3-tapper, then -2, -3… function autoRunName(model, level, persona) { const base = `${slug(model)}-L${level}-${slug(persona)}`; if (!existsSync(runPath(base))) return base; for (let n = 2; n < 99; n++) if (!existsSync(runPath(`${base}-${n}`))) return `${base}-${n}`; return `${base}-${Date.now()}`; } const portOpen = () => new Promise(done => { const socket = createConnection({ port: PORT, host: '127.0.0.1' }); const finish = ok => { socket.destroy(); done(ok); }; socket.setTimeout(400); socket.on('connect', () => finish(true)); socket.on('error', () => finish(false)); socket.on('timeout', () => finish(false)); }); // Bring the observer page within reach: start the static server if it is not // already up, then print (and on macOS open) the URL for this run. Returns // null in a portable pack, which ships no page and no server — a URL there // would point at whatever else happens to hold the port. const OBSERVER = resolve('labs/bench/index.html'), SERVER = resolve('scripts/serve.mjs'); async function observerUrl(run, { open = false } = {}) { if (!existsSync(OBSERVER) || !existsSync(SERVER)) return null; const url = `http://localhost:${PORT}/labs/bench/index.html${run ? `?run=${encodeURIComponent(run)}&live=1` : ''}`; if (!(await portOpen())) { const server = spawn(process.execPath, [SERVER], { detached: true, stdio: 'ignore' }); server.unref(); for (let i = 0; i < 20 && !(await portOpen()); i++) await new Promise(r => setTimeout(r, 150)); } if (open && process.platform === 'darwin') spawnSync('open', [url], { stdio: 'ignore' }); return url; } const loadRun = () => { if (!flags.run) fail('需要 --run=NAME'); if (!existsSync(runPath(flags.run))) fail(`没有这局:${flags.run}(先 start)`); return JSON.parse(readFileSync(runPath(flags.run), 'utf8')); }; const saveRun = runObj => { mkdirSync(RUNS, { recursive: true }); runObj.updatedAt = new Date().toISOString(); writeFileSync(runPath(runObj.run), JSON.stringify(runObj)); const indexPath = resolve(RUNS, 'index.json'); const index = existsSync(indexPath) ? JSON.parse(readFileSync(indexPath, 'utf8')) : []; const entry = { run: runObj.run, level: runObj.level.id, persona: runObj.persona, model: runObj.model, peek: runObj.peek, status: runObj.status, steps: runObj.steps.length, updatedAt: runObj.updatedAt }; const at = index.findIndex(e => e.run === runObj.run); if (at >= 0) index[at] = entry; else index.push(entry); writeFileSync(indexPath, JSON.stringify(index, null, 1)); }; if (command === 'list') { const set = flags.set || 'levels/prototypes/d5.json'; const levels = loadSet(set); console.log(`${set} · ${levels.length} 关`); levels.forEach((l, i) => console.log(` ${String(i + 1).padStart(3)} ${l.id} ${l.cells.length} 格 · ${l.tiles.length} 枚${l.prototype ? ` · 锁住 ${Math.round(l.prototype.stats.lockedShare * 100)}% · 最少 ${l.prototype.stats.rotations} 转` : ''}`)); } else if (command === 'rules') { console.log(rulesText(!!flags.peek)); } else if (command === 'start') { const levels = loadSet(flags.set || 'levels/prototypes/d5.json'); const wanted = flags.level ?? 1; const level = levels.find(l => l.id === wanted) || levels[Number(wanted) - 1]; if (!level) fail(`找不到关卡 ${wanted}(本组共 ${levels.length} 关,用 list 查看)`); const persona = flags.persona || 'tapper'; const model = flags.model || process.env.BENCH_MODEL || 'unknown'; if (model === 'unknown') console.warn('提示:没有 --model,本局记为 unknown;填上你的模型标识才能按模型比较。'); const name = flags.run ? String(flags.run) : autoRunName(model, levels.indexOf(level) + 1, persona); if (!/^[\w.-]+$/.test(name)) fail('run 名只能用字母、数字、_ - .'); if (existsSync(runPath(name)) && !flags.force) fail(`${name} 已存在;换个名字或加 --force`); const agent = flags.agent ? String(flags.agent) : ''; const runObj = createRun({ level, run: name, persona, model, agent, peek: !!flags.peek, budget: Number(flags.budget || 80), startedAt: new Date().toISOString() }); saveRun(runObj); const url = await observerUrl(name, { open: !!flags.open }); console.log(`开始 ${runObj.run} · 关卡 ${level.id} · 人设 ${runObj.persona} · 模型 ${runObj.model}${runObj.agent ? ` · ${runObj.agent}` : ''} · 预算 ${runObj.budget} 步 · peek ${runObj.peek ? '开' : '关'}`); if (!runObj.agent) console.warn('提示:没有 --agent,观察页横幅只会显示模型名。填上运行环境和模式,例如 --agent="Claude Code · ultracode"。'); if (url) console.log(`观察页(人类实时观看):${url}`); console.log(`后续命令都用 --run=${runObj.run}`); console.log(''); // --no-rules: an agent that already read AGENT.md skips the rules block. if (!flags['no-rules']) { console.log(rulesText(runObj.peek)); console.log(''); } console.log(observe(runObj, { format: fmt() })); } else if (command === 'open') { const url = await observerUrl(flags.run ? String(flags.run) : null, { open: flags.open !== 'false' }); console.log(url || '这个副本没有观察页(便携包);把 runs/ 交回原仓库用 bench-import.mjs 导入后再看。'); } else if (command === 'observe') { const runObj = loadRun(); const view = observe(runObj, { format: fmt() }); console.log(typeof view === 'string' ? view : JSON.stringify(view, null, 1)); } else if (command === 'act') { const runObj = loadRun(); // One action, or several separated by commas: `tap t4, tap t5, rotate right`. // A chain stops at the first tap that fails and after any turn. const parse = words => { const [kind, ...rest] = words; if (kind === 'tap' || kind === 'peek') return { type: kind, id: rest[0] }; if (kind === 'rotate') return { type: 'rotate', direction: ['right', 'r', '1', '+1', '右'].includes(rest[0]) ? 1 : ['left', 'l', '-1', '左'].includes(rest[0]) ? -1 : 0 }; if (kind === 'undo') return { type: 'undo' }; if (kind === 'giveup') return { type: 'giveup', reason: rest.join(' ') }; fail('act 需要:tap ID | rotate left|right | undo | peek ID | giveup 原因;多个动作用逗号隔开'); }; const groups = positional.join(' ').split(',').map(s => s.trim()).filter(Boolean).map(s => parse(s.split(/\s+/))); if (!groups.length) fail('act 需要一个动作'); if (groups.length > 8) fail('一次最多连做 8 个动作'); const { outcomes, stopped } = actMany(runObj, groups, { note: flags.note ? String(flags.note) : '' }); saveRun(runObj); for (const out of outcomes) console.log(`${describeAction(out.step?.action || out.rejected?.action)} → ${out.feedback}`); if (stopped) console.log(`(${stopped})`); if (!flags.quiet && runObj.status === 'playing') { console.log(''); console.log(observe(runObj, { format: fmt() === 'json' ? 'text' : fmt() })); } if (runObj.status !== 'playing') console.log(`\n本局结束:${runObj.status}。请用 rate 提交评分。`); if (flags.json) console.log(JSON.stringify({ outcomes, stopped })); } else if (command === 'rate') { const runObj = loadRun(); let answers; try { answers = JSON.parse(flags.json || '{}'); } catch { fail('--json 需要合法 JSON'); } const rating = rate(runObj, answers); saveRun(runObj); console.log('已记录评分:' + JSON.stringify(rating)); const missing = RATING_QUESTIONS.filter(q => rating[q.key] === null).map(q => q.key); if (missing.length) console.log('未填:' + missing.join(', ')); // Close the loop for the agent: the numbers it should report, and the link. const m = metrics(runObj); const secs = ms => ms == null ? '—' : ms >= 1000 ? `${(ms / 1000).toFixed(1)} s` : `${ms} ms`; console.log(`本局:${m.status} · ${m.steps} 步 · ${m.decisions} 次决策 · 空点 ${m.deadTaps}(擦碰 ${m.grazeDeadTaps})· 旋转 ${m.rotations}/${m.minimumRotations ?? '?'} · 耗时 ${secs(m.timing.totalMs)}`); const replayUrl = await observerUrl(runObj.run); if (replayUrl) console.log(`回放:${replayUrl}`); } else if (command === 'status') { const runObj = loadRun(); console.log(`${runObj.run} · ${runObj.level.id} · ${runObj.status} · ${runObj.steps.length}/${runObj.budget} 步 · 人设 ${runObj.persona} · 模型 ${runObj.model}`); for (const s of runObj.steps) console.log(` ${String(s.n).padStart(3)} ${describeAction(s.action).padEnd(10)} → ${describeResult(s.result)}${s.note ? ` 「${s.note}」` : ''}`); } else if (command === 'metrics') { const runObj = loadRun(); const m = metrics(runObj); if (flags.json) console.log(JSON.stringify(m, null, 1)); else console.log(Object.entries(m).filter(([k]) => k !== 'rating').map(([k, v]) => `${k}: ${typeof v === 'number' ? Number(v.toFixed(2)) : v}`).join('\n') + (m.rating ? '\nrating: ' + JSON.stringify(m.rating) : '')); } else if (command === 'runs') { const dir = existsSync(RUNS) ? readdirSync(RUNS).filter(f => f.endsWith('.json') && f !== 'index.json') : []; for (const f of dir) { const r = JSON.parse(readFileSync(resolve(RUNS, f), 'utf8')); console.log(`${r.run.padEnd(28)} ${r.level.id.padEnd(34)} ${r.status.padEnd(9)} ${String(r.steps.length).padStart(3)} 步 ${r.persona}/${r.model}`); } } else { fail('用法:node scripts/bench.mjs list|start|act|observe|rate|status|metrics|runs|open|rules …\n完整说明(给 Agent 读的那一份):labs/bench/AGENT.md'); }