feat(bench): batch runners, aggregation and a standalone kit
Four ways in, depending on what you have: a shell agent driving bench.mjs, a direct-API loop for models with only an endpoint (plain text in, one JSON line out, so weak and local models can play), a rule-following bot that sweeps 1000 levels in 12 seconds, and a folder you can hand to someone else. bench-kit.mjs builds that folder: CLI, engine, level pack, observer page and tile atlas, about 1.5 MB, needing only Node 20 — no repo, no install, no network. The engine files are discovered by following imports so a new module can't be left behind, every path rewrite is asserted so a rename fails the build loudly, and the builder plays a real game inside the result before declaring it done. bench-report.mjs aggregates two ways: per level (is this level any good) and per model (how does this agent play). bench-import.mjs brings runs played elsewhere back in under a tag. bench-pack.mjs is the earlier, smaller export with no observer page; superseded by bench-kit but kept in history rather than dropped silently. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
cabinw
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env node
|
||||
// Direct-API agent loop: the fast path for batch runs. Plain text in, one JSON
|
||||
// line (or a JSON array to chain several actions) out — no tool-use API needed,
|
||||
// so weak and local models can play. The compact system prompt is sent once.
|
||||
// BENCH_PROVIDER=anthropic ANTHROPIC_API_KEY=… node scripts/bench-agent.mjs --run=r1 --level=1 --model=claude-sonnet-5 --persona=tapper
|
||||
// BENCH_PROVIDER=openai OPENAI_API_KEY=… node scripts/bench-agent.mjs --run=r2 --level=1 --model=gpt-4o-mini
|
||||
// BENCH_PROVIDER=openai BENCH_BASE_URL=http://localhost:11434/v1 node scripts/bench-agent.mjs --run=r3 --level=1 --model=qwen2.5:7b
|
||||
// BENCH_PROVIDER=mock node scripts/bench-agent.mjs --run=r4 --level=1 # rule-following bot, no network
|
||||
// --dry prints the first prompt and exits · --full-protocol sends the whole AGENT.md instead of the compact rules
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { createRun, observe, actMany, rate, rulesText, RATING_QUESTIONS, describeAction, metrics } from '../src/bench.js';
|
||||
|
||||
const flags = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
|
||||
// Presets for the endpoints we know; anything else OpenAI-compatible works via
|
||||
// BENCH_PROVIDER=openai + BENCH_BASE_URL. Model ids change — pass --model.
|
||||
const PROVIDERS = {
|
||||
anthropic: { style: 'anthropic', base: 'https://api.anthropic.com', keyVar: 'ANTHROPIC_API_KEY', model: 'claude-sonnet-5' },
|
||||
openai: { style: 'openai', base: 'https://api.openai.com/v1', keyVar: 'OPENAI_API_KEY', model: 'gpt-4o-mini' },
|
||||
deepseek: { style: 'openai', base: 'https://api.deepseek.com', keyVar: 'DEEPSEEK_API_KEY', model: 'deepseek-flash' },
|
||||
xai: { style: 'openai', base: 'https://api.x.ai/v1', keyVar: 'XAI_API_KEY', model: 'grok-4.6' },
|
||||
mock: { style: 'mock', base: '', keyVar: '', model: 'rulebot' },
|
||||
};
|
||||
const providerName = process.env.BENCH_PROVIDER || 'anthropic';
|
||||
const provider = PROVIDERS[providerName];
|
||||
if (!provider) { console.error(`未知 BENCH_PROVIDER=${providerName}，可用：${Object.keys(PROVIDERS).join(' / ')}`); process.exit(1); }
|
||||
const model = flags.model || process.env.BENCH_MODEL || provider.model;
|
||||
const baseUrl = process.env.BENCH_BASE_URL || provider.base;
|
||||
const apiKey = provider.keyVar ? process.env[provider.keyVar] || '' : '';
|
||||
const RUNS = resolve('bench/runs');
|
||||
if (!flags.run || !flags.level) { console.error('需要 --run=NAME --level=ID|序号'); process.exit(1); }
|
||||
// A missing key otherwise surfaces as a raw 401 stack halfway through a batch.
|
||||
if (provider.keyVar && !apiKey && !process.env.BENCH_BASE_URL && !flags.dry) {
|
||||
console.error(`${providerName} 需要环境变量 ${provider.keyVar}。本地端点用 BENCH_PROVIDER=openai BENCH_BASE_URL=…，无网络试跑用 BENCH_PROVIDER=mock。`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const loadSet = path => { const raw = JSON.parse(readFileSync(resolve(path), 'utf8')); return Array.isArray(raw) ? raw : Array.isArray(raw.levels) ? raw.levels : Object.values(raw).filter(l => l && l.cells); };
|
||||
const levels = loadSet(flags.set || 'levels/prototypes/d5.json');
|
||||
const level = levels.find(l => l.id === flags.level) || levels[Number(flags.level) - 1];
|
||||
if (!level) { console.error(`找不到关卡 ${flags.level}`); process.exit(1); }
|
||||
const runObj = createRun({ level, run: flags.run, persona: flags.persona || 'tapper', model, agent: flags.agent ? String(flags.agent) : `${providerName} API`, peek: !!flags.peek, budget: Number(flags.budget || 80), startedAt: new Date().toISOString() });
|
||||
runObj.timing = { decisions: [], totalMs: 0 };
|
||||
// The run file is ours alone; the shared index is only touched at start and
|
||||
// end (parallel runs would otherwise lose each other's entries), and the
|
||||
// batch runner rebuilds it from the directory anyway.
|
||||
function save(withIndex = false) {
|
||||
mkdirSync(RUNS, { recursive: true });
|
||||
runObj.updatedAt = new Date().toISOString();
|
||||
writeFileSync(resolve(RUNS, `${runObj.run}.json`), JSON.stringify(runObj));
|
||||
if (!withIndex) return;
|
||||
const indexPath = resolve(RUNS, 'index.json');
|
||||
let index = [];
|
||||
try { index = existsSync(indexPath) ? JSON.parse(readFileSync(indexPath, 'utf8')) : []; } catch { index = []; }
|
||||
const entry = { run: runObj.run, level: level.id, persona: runObj.persona, model, peek: runObj.peek, status: runObj.status, steps: runObj.steps.length, updatedAt: runObj.updatedAt };
|
||||
const at = index.findIndex(e => e.run === runObj.run); if (at >= 0) index[at] = entry; else index.push(entry);
|
||||
writeFileSync(indexPath, JSON.stringify(index, null, 1));
|
||||
}
|
||||
|
||||
const PERSONAS = {
|
||||
reader: '你是读盘型玩家：先把路线看清楚，想好了再点，尽量少试错。',
|
||||
tapper: '你是试错型玩家：看个大概就点，用反馈学习；被挡了再想。',
|
||||
turner: '你是转盘型玩家：不确定的时候先转一下看看打开了什么。',
|
||||
novice: '你是第一次玩的新手，没有固定策略，按直觉来。',
|
||||
};
|
||||
const ACTION_FORMAT = [
|
||||
'输出格式：每一轮只输出 JSON，不要任何别的文字。一个动作是一个对象；连做几步就输出一个数组（最多 8 个），链在第一个没成功的动作和任何一次旋转之后自动停下并返回新盘面。',
|
||||
'{"act":"tap","id":"t5","note":"为什么"} {"act":"rotate","direction":"right"|"left","note":"…"} {"act":"undo","note":"…"}' + (runObj.peek ? ' {"act":"peek","id":"t3","note":"…"}' : '') + ' {"act":"giveup","reason":"…"}',
|
||||
'[{"act":"tap","id":"t4","note":"边上"},{"act":"tap","id":"t5","note":"边上"},{"act":"rotate","direction":"right","note":"清完就转"}]',
|
||||
'像人一样：看清一次盘面就把有把握的几步一起提交，别每点一枚重新读一遍；旋转之后必须重新看盘再决定。每个动作都带 note。',
|
||||
].join('\n');
|
||||
const system = flags['full-protocol']
|
||||
? ['你正在玩一关六角轨道解谜。下面是操作协议、规则和你的人设。', '', readFileSync(resolve('labs/bench/AGENT.md'), 'utf8'), '', rulesText(runObj.peek), '', PERSONAS[runObj.persona] || PERSONAS.novice, '', ACTION_FORMAT].join('\n')
|
||||
: ['你正在玩一关六角轨道解谜，目标是让所有棋子离场。', '', rulesText(runObj.peek), '', PERSONAS[runObj.persona] || PERSONAS.novice, '', ACTION_FORMAT].join('\n');
|
||||
|
||||
async function chat(messages) {
|
||||
if (provider.style === 'mock') return mockPlayer(messages);
|
||||
if (provider.style === 'anthropic') {
|
||||
const res = await fetch(`${baseUrl}/v1/messages`, { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey, 'anthropic-version': '2023-06-01' },
|
||||
body: JSON.stringify({ model, max_tokens: 600, system, messages }) });
|
||||
const data = await res.json();
|
||||
if (!res.ok) throw new Error(JSON.stringify(data));
|
||||
return data.content.map(c => c.text || '').join('');
|
||||
}
|
||||
// Only the parameters every OpenAI-compatible endpoint we target accepts —
|
||||
// DeepSeek rejects frequency_penalty / presence_penalty outright.
|
||||
const res = await fetch(`${baseUrl}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey || 'none'}` },
|
||||
body: JSON.stringify({ model, max_tokens: 600, temperature: 0.3, messages: [{ role: 'system', content: system }, ...messages] }) });
|
||||
const data = await res.json();
|
||||
if (!res.ok) throw new Error(JSON.stringify(data));
|
||||
const choice = data.choices?.[0]?.message;
|
||||
if (!choice) throw new Error(JSON.stringify(data));
|
||||
return choice.content || '';
|
||||
}
|
||||
// Rule bot: plays exactly the written advice — tap everything whose line is
|
||||
// clear, otherwise turn right. A floor for the LLM runs and a fast sweep tool.
|
||||
function mockPlayer() {
|
||||
const view = observe(runObj, { format: 'json' });
|
||||
const clear = view.pieces.filter(p => p.route.end === 'exit' && !p.route.onRoute.length && !p.route.grazes.length).slice(0, 8);
|
||||
if (clear.length) return JSON.stringify(clear.map(p => ({ act: 'tap', id: p.id, note: '路上没有棋子也不紧贴谁' })));
|
||||
return JSON.stringify({ act: 'rotate', direction: 'right', note: '一枚都走不了，转一下' });
|
||||
}
|
||||
const firstJson = text => { const m = text.match(/\{[\s\S]*?\}/); if (!m) return null; try { return JSON.parse(m[0]); } catch { return null; } };
|
||||
const toAction = j => {
|
||||
if (!j || !j.act) return null;
|
||||
if (j.act === 'rotate') return { type: 'rotate', direction: ['right', 'r', 1, '1', '+1', '右'].includes(j.direction) ? 1 : -1, note: j.note || '' };
|
||||
if (j.act === 'tap' || j.act === 'peek') return { type: j.act, id: String(j.id || ''), note: j.note || '' };
|
||||
if (j.act === 'undo') return { type: 'undo', note: j.note || '' };
|
||||
if (j.act === 'giveup') return { type: 'giveup', reason: j.reason || j.note || '', note: j.note || '' };
|
||||
return null;
|
||||
};
|
||||
const parseActions = text => {
|
||||
const arr = text.match(/\[[\s\S]*\]/);
|
||||
if (arr) { try { const list = JSON.parse(arr[0]); if (Array.isArray(list)) return list.map(toAction).filter(Boolean).slice(0, 8); } catch { /* fall through */ } }
|
||||
const one = toAction(firstJson(text));
|
||||
return one ? [one] : [];
|
||||
};
|
||||
|
||||
save(true);
|
||||
const keep = Number(flags.context || 6);
|
||||
const history = [];
|
||||
let prompt = `${observe(runObj)}\n\n请输出 JSON 动作（或数组连做几步）。`;
|
||||
if (flags.dry) { console.log('=== system ===\n' + system + '\n\n=== user ===\n' + prompt); process.exit(0); }
|
||||
const started = Date.now();
|
||||
console.log(`开始 ${runObj.run} · ${level.id} · ${providerName}/${model} · 人设 ${runObj.persona}`);
|
||||
while (runObj.status === 'playing') {
|
||||
history.push({ role: 'user', content: prompt });
|
||||
const window = history.slice(-keep * 2);
|
||||
let actions = [], reply = '';
|
||||
const t0 = Date.now();
|
||||
for (let attempt = 0; attempt < 3 && !actions.length; attempt++) {
|
||||
reply = await chat(attempt ? [...window, { role: 'assistant', content: reply }, { role: 'user', content: '上一条不是合法的 JSON 动作。只输出 JSON，例如 {"act":"tap","id":"t3","note":"…"} 或一个数组。' }] : window);
|
||||
actions = parseActions(reply);
|
||||
}
|
||||
const thinkMs = Date.now() - t0;
|
||||
if (!actions.length) { console.error('连续三次无法解析动作，停止。最后回复：' + reply); break; }
|
||||
history.push({ role: 'assistant', content: JSON.stringify(actions.map(a => ({ act: a.type, ...(a.id ? { id: a.id } : {}), ...(a.direction ? { direction: a.direction } : {}), note: a.note }))) });
|
||||
const { outcomes, stopped } = actMany(runObj, actions);
|
||||
runObj.timing.decisions.push({ step: runObj.steps.length, actions: outcomes.length, thinkMs });
|
||||
save();
|
||||
const lines = outcomes.map(out => `${describeAction(out.step?.action || out.rejected?.action)} → ${out.feedback}`);
|
||||
for (const [i, out] of outcomes.entries()) console.log(` ${String(out.step?.n ?? '-').padStart(3)} ${lines[i]}${i === 0 ? ` [${thinkMs} ms]` : ''}${actions[i]?.note ? ` 「${actions[i].note}」` : ''}`);
|
||||
if (stopped) { console.log(` （${stopped}）`); lines.push(`（${stopped}）`); }
|
||||
prompt = `${lines.join('\n')}\n\n${observe(runObj)}\n\n请输出 JSON 动作（或数组连做几步）。`;
|
||||
}
|
||||
if (runObj.status !== 'playing') {
|
||||
const ask = `本局结束（${runObj.status}）。请评分，只输出一行 JSON：` + RATING_QUESTIONS.map(q => `${q.key}=${q.text}`).join('；');
|
||||
// The rule bot has no opinion; leaving rating null keeps it out of the averages.
|
||||
if (provider.style !== 'mock') { const reply = await chat([...history.slice(-keep * 2), { role: 'user', content: ask }]); rate(runObj, firstJson(reply) || {}); }
|
||||
}
|
||||
runObj.timing.totalMs = Date.now() - started;
|
||||
save(true);
|
||||
const m = metrics(runObj);
|
||||
console.log(`结束：${runObj.status} · ${m.steps} 步 · ${m.decisions} 次决策 · 空点 ${m.deadTaps} · 旋转 ${m.rotations}/${m.minimumRotations ?? '?'} · ${(runObj.timing.totalMs / 1000).toFixed(1)} s · bench/runs/${runObj.run}.json`);
|
||||
Reference in New Issue
Block a user