cabinwandClaude Opus 5
f3e91e3597
Four ways in, depending on what you have: a shell agent driving bench.mjs, a direct-API loop for models with only an endpoint (plain text in, one JSON line out, so weak and local models can play), a rule-following bot that sweeps 1000 levels in 12 seconds, and a folder you can hand to someone else. bench-kit.mjs builds that folder: CLI, engine, level pack, observer page and tile atlas, about 1.5 MB, needing only Node 20 — no repo, no install, no network. The engine files are discovered by following imports so a new module can't be left behind, every path rewrite is asserted so a rename fails the build loudly, and the builder plays a real game inside the result before declaring it done. bench-report.mjs aggregates two ways: per level (is this level any good) and per model (how does this agent play). bench-import.mjs brings runs played elsewhere back in under a tag. bench-pack.mjs is the earlier, smaller export with no observer page; superseded by bench-kit but kept in history rather than dropped silently. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
63 lines
4.4 KiB
JavaScript
63 lines
4.4 KiB
JavaScript
#!/usr/bin/env node
|
||
// Batch runner: many levels × personas through the direct-API loop, N at a time.
|
||
// BENCH_PROVIDER=anthropic ANTHROPIC_API_KEY=… node scripts/bench-batch.mjs --levels=1-18 --personas=tapper,reader --model=claude-sonnet-5 --concurrency=6
|
||
// BENCH_PROVIDER=mock node scripts/bench-batch.mjs --set=levels/v0.5.json --levels=all --concurrency=8 # rule bot over the whole campaign
|
||
// Options: --set=… (default levels/prototypes/d5.json) --levels=all|1-18|1,4,7 --personas=… --model=… --budget=80 --peek --prefix=NAME --concurrency=4 --force
|
||
// Runs already finished (bench/runs/<prefix>-L<n>-<persona>.json with status ≠ playing) are skipped unless --force.
|
||
import { spawn } from 'node:child_process';
|
||
import { existsSync, readFileSync } from 'node:fs';
|
||
import { resolve } from 'node:path';
|
||
import { rebuildIndex } from './bench-index.mjs';
|
||
|
||
const flags = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
|
||
const set = flags.set || 'levels/prototypes/d5.json';
|
||
const raw = JSON.parse(readFileSync(resolve(set), 'utf8'));
|
||
const levels = Array.isArray(raw) ? raw : Array.isArray(raw.levels) ? raw.levels : Object.values(raw).filter(l => l && l.cells);
|
||
const spec = String(flags.levels || 'all');
|
||
const indices = spec === 'all' ? levels.map((_, i) => i + 1)
|
||
: spec.split(',').flatMap(part => { const m = part.match(/^(\d+)-(\d+)$/); return m ? Array.from({ length: Number(m[2]) - Number(m[1]) + 1 }, (_, i) => Number(m[1]) + i) : [Number(part)]; }).filter(n => n >= 1 && n <= levels.length);
|
||
const personas = String(flags.personas || 'tapper').split(',');
|
||
const provider = process.env.BENCH_PROVIDER || 'anthropic';
|
||
// Same presets as bench-agent.mjs; --model overrides.
|
||
const DEFAULT_MODEL = { mock: 'rulebot', anthropic: 'claude-sonnet-5', openai: 'gpt-4o-mini', deepseek: 'deepseek-flash', xai: 'grok-4.6' };
|
||
const model = flags.model || process.env.BENCH_MODEL || DEFAULT_MODEL[provider] || 'unknown';
|
||
const prefix = flags.prefix || (provider === 'mock' ? 'bot' : model.replace(/[^\w.-]+/g, '-'));
|
||
const concurrency = Number(flags.concurrency || 4);
|
||
|
||
const jobs = [];
|
||
for (const n of indices) for (const persona of personas) {
|
||
const run = `${prefix}-L${n}-${persona}`;
|
||
const file = resolve('bench/runs', `${run}.json`);
|
||
if (existsSync(file) && !flags.force) { try { const r = JSON.parse(readFileSync(file, 'utf8')); if (r.status !== 'playing') continue; } catch { /* rerun */ } }
|
||
jobs.push({ run, n, persona });
|
||
}
|
||
console.log(`${set} · ${indices.length} 关 × ${personas.length} 人设 → ${jobs.length} 局待跑（已完成的已跳过）· ${provider}/${model} · 并发 ${concurrency}`);
|
||
const started = Date.now(), total = jobs.length;
|
||
let done = 0, failed = 0;
|
||
const results = [];
|
||
async function worker() {
|
||
while (jobs.length) {
|
||
const job = jobs.shift();
|
||
const args = ['scripts/bench-agent.mjs', `--run=${job.run}`, `--level=${job.n}`, `--set=${set}`, `--persona=${job.persona}`, `--model=${model}`];
|
||
if (flags.budget) args.push(`--budget=${flags.budget}`);
|
||
if (flags.agent) args.push(`--agent=${flags.agent}`);
|
||
if (flags.peek) args.push('--peek');
|
||
if (flags['full-protocol']) args.push('--full-protocol');
|
||
const t0 = Date.now();
|
||
const code = await new Promise(res => { const p = spawn('node', args, { stdio: ['ignore', 'pipe', 'pipe'] }); let tail = ''; p.stdout.on('data', d => { tail = (tail + d).slice(-400); }); p.stderr.on('data', d => { tail = (tail + d).slice(-400); }); p.on('close', c => { job.tail = tail.trim().split('\n').pop(); res(c); }); });
|
||
done++;
|
||
if (code !== 0) failed++;
|
||
results.push({ ...job, code, seconds: (Date.now() - t0) / 1000 });
|
||
console.log(`[${done}/${total}] ${job.run.padEnd(30)} ${code === 0 ? job.tail : `失败(exit ${code}) ${job.tail}`}`);
|
||
}
|
||
}
|
||
// Children only write their own run file; the shared index is rebuilt here so
|
||
// the observer page can list runs while the batch is still going.
|
||
const indexTimer = setInterval(() => { try { rebuildIndex(); } catch { /* next tick */ } }, 5000);
|
||
await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(1, jobs.length)) }, worker));
|
||
clearInterval(indexTimer);
|
||
rebuildIndex();
|
||
const elapsed = (Date.now() - started) / 1000;
|
||
console.log(`\n完成 ${done} 局，失败 ${failed}，用时 ${elapsed.toFixed(0)} s（平均 ${(elapsed / Math.max(1, done)).toFixed(1)} s/局，并发 ${concurrency}）`);
|
||
console.log('汇总：node scripts/bench-report.mjs');
|