feat(bench): agent bench session core and CLI
Let any agent with a shell play a level the way a person does: look at the board, decide what to tap or which way to turn, read the feedback, decide again. Every step carries the agent's own reason, so a human can watch and judge the level from how it was played. observe() returns what the eye sees — each piece's drawn route, which cells it crosses, who is sitting on it, and who it brushes past — never "which pieces can leave". act() goes through the real Game rules; metrics() scores the run afterwards with the solver. Pieces are wide, so a route with nobody standing on it can still be blocked by a neighbour the curve hugs. That is invisible in a cell listing, so every obstacle is reported, on-route and grazing alike: reporting only the leading one hid the rest, and in 113 of 581 such cases clearing the blocker sprang a collision the agent had never been shown. AGENT.md is the single file an agent reads — rules, board reading, commands, rating scale, worked example. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
cabinw
+292
@@ -0,0 +1,292 @@
|
||||
import { DIRS, Game, available, clone, opposite, rotateBoard, solveRotations, trace } from './core.js';
|
||||
|
||||
// Agent bench: a level session that any agent can drive through observe/act.
|
||||
// The observation carries only what a person can read off the screen — cells,
|
||||
// rails, pieces, arrows, orientation. Never which pieces can currently leave,
|
||||
// and never the solver. Metrics use the solver afterwards, on the recorded run.
|
||||
|
||||
export const PORT_NAMES = ['↑', '↗', '↘', '↓', '↙', '↖'];
|
||||
export const ACTIONS = ['tap', 'rotate', 'undo', 'peek', 'giveup'];
|
||||
const countBends = route => route.steps.filter(s => s.enter !== null && opposite(s.enter) !== s.exit).length;
|
||||
|
||||
export function rulesText(peek) {
|
||||
return [
|
||||
'规则：棋盘由六角格组成，每格画着轨道。点击一枚棋子，它沿着自己箭头所指的那条轨道一路走，直到走出棋盘边缘（离场）。途中如果撞上另一枚棋子，就退回原位（棋子有宽度，擦到紧邻的棋子也算撞）。被挡不扣任何东西。',
|
||||
'旋转底盘：整块棋盘连轨道转 60°，棋子的位置和箭头不动——于是每枚棋子接上的是另一条轨道，路线全部改变。左转右转免费、可反复、可撤销；转 6 次回到原样。',
|
||||
'目标：让所有棋子离场。没有失败；有步数预算。',
|
||||
'',
|
||||
'怎么看盘面：观察里给出每枚棋子当前的路线——它会依次经过哪些格、在哪里出界（或绕成闭环）。路线上压着别的棋子，标成 [tX]；路线紧贴着别的棋子（棋子有宽度，会擦到），标成"紧贴 tX"。这就是人眼在屏幕上看到的那条线。',
|
||||
'怎么玩：',
|
||||
'1) 路上没有棋子、不紧贴谁、而且能出界的，点它就能离场。先点这些。',
|
||||
'2) 路上有 [tX] 或紧贴 tX 的，得先让 tX 走：tX 自己能走就先点 tX；tX 也被挡就顺着往下找，先解开最外层的。',
|
||||
'3) 每条路线都有人挡、或都是闭环——一枚都走不了，那就旋转。旋转后所有路线都变，重新看。转完没有出口，可以再转或转回去。',
|
||||
'4) 每一步都算预算：先做有把握的，别乱试。' + (peek ? ' peek 与观察里的路线是同一信息，一般用不着。' : ''),
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
// Routes are shown in full: a folded "…(共 54 格)" hides exactly the part a
|
||||
// careful player would re-check. Callers that must save tokens pass routeLimit.
|
||||
const ROUTE_SHOW = Infinity;
|
||||
// The drawn line, as a person sees it: the cells it passes, who is sitting on
|
||||
// it, and — because pieces are wide — who it brushes past. The brush is taken
|
||||
// from the real swept collider, since that is what the eye judges from the
|
||||
// curve hugging a neighbour; a cell-adjacency guess would over-warn.
|
||||
function pieceRoutes(game) {
|
||||
const occupant = new Map(game.tiles.map(t => [`${t.q},${t.r}`, t.id]));
|
||||
return game.tiles.map(t => {
|
||||
const geo = trace(game.level, t, [], true);
|
||||
const cells = geo.steps.map(s => { const who = occupant.get(`${s.q},${s.r}`); return { q: s.q, r: s.r, piece: who && who !== t.id ? who : null }; });
|
||||
const onRoute = [...new Set(cells.filter(c => c.piece).map(c => c.piece))];
|
||||
// Every piece that would stop this one, not just the first. Reporting only
|
||||
// the leading blocker hid every graze behind it: in 28% of "blocked by a
|
||||
// piece on the line" cases, clearing that piece sprang a collision the
|
||||
// agent had never been shown. Test each candidate on its own so nothing
|
||||
// masks anything else.
|
||||
const grazes = [];
|
||||
for (const other of game.tiles) {
|
||||
if (other.id === t.id || onRoute.includes(other.id)) continue;
|
||||
const hit = trace(game.level, t, [other]);
|
||||
if (hit.status === 'blocked' && hit.blocker === other.id) grazes.push({ id: other.id, at: hit.steps.length });
|
||||
}
|
||||
grazes.sort((a, b) => a.at - b.at);
|
||||
return { id: t.id, q: t.q, r: t.r, dir: t.dir, arrow: PORT_NAMES[t.dir], color: t.color,
|
||||
route: { end: geo.status === 'exit' ? 'exit' : 'loop', cells: cells.slice(1), bends: countBends(geo), onRoute, grazes: grazes.map(g => g.id) } };
|
||||
});
|
||||
}
|
||||
function routeLine(p) {
|
||||
const shown = p.route.cells.slice(0, ROUTE_SHOW).map(c => `${c.q},${c.r}${c.piece ? `[${c.piece}]` : ''}`);
|
||||
const tail = p.route.cells.length > ROUTE_SHOW ? ` … (共 ${p.route.cells.length} 格)` : '';
|
||||
const end = p.route.end === 'exit' ? '出界' : '闭环';
|
||||
const who = p.route.onRoute.length ? `路上有 ${p.route.onRoute.join(',')}` : '路上没有棋子';
|
||||
const brush = p.route.grazes.length ? `${p.route.onRoute.length ? '，另外' : '，但'}紧贴 ${p.route.grazes.join(',')}（会擦到）` : '';
|
||||
if (!p.route.cells.length) return `${p.id} @ ${p.q},${p.r} ${p.arrow}：就在边上，直接出界（0 格）· 路上没有棋子${brush}`;
|
||||
return `${p.id} @ ${p.q},${p.r} ${p.arrow}：${shown.join(' → ')}${tail} → ${end}（${p.route.cells.length} 格 · ${p.route.bends} 弯）· ${who}${brush}`;
|
||||
}
|
||||
|
||||
// `model` is the machine-readable id used for grouping; `agent` is whatever the
|
||||
// agent says about itself — harness, mode, reasoning effort — shown as the
|
||||
// banner over the board so a watcher knows who is playing.
|
||||
export function createRun({ level, run, persona = 'default', model = 'unknown', agent = '', peek = false, budget = 80, startedAt = null }) {
|
||||
if (!level?.cells || !level?.tiles) throw new Error('createRun needs a level with cells and tiles');
|
||||
return { version: 1, run, persona, model, agent: String(agent || '').slice(0, 120), peek, budget, startedAt, updatedAt: startedAt, status: 'playing', level: clone(level), steps: [], rating: null, reason: null };
|
||||
}
|
||||
|
||||
// Rebuild the live game by replaying the recorded actions.
|
||||
export function replay(runObj) {
|
||||
const game = new Game(runObj.level);
|
||||
for (const step of runObj.steps) {
|
||||
if (step.action.type === 'tap') game.tap(step.action.id);
|
||||
else if (step.action.type === 'rotate') game.rotate(step.action.direction);
|
||||
else if (step.action.type === 'undo') game.undo();
|
||||
}
|
||||
return game;
|
||||
}
|
||||
|
||||
function pairs(tracks) {
|
||||
const out = [];
|
||||
for (let a = 0; a < 6; a++) if (a < tracks[a]) out.push(`${a}-${tracks[a]}`);
|
||||
return out.join(' ');
|
||||
}
|
||||
|
||||
export function observe(runObj, { format = 'text' } = {}) {
|
||||
const game = replay(runObj);
|
||||
const last = runObj.steps[runObj.steps.length - 1] || null;
|
||||
const state = {
|
||||
run: runObj.run, level: runObj.level.id, status: runObj.status, step: runObj.steps.length, budget: runObj.budget,
|
||||
orientation: game.orientation, rotationsUsed: game.rotations, remaining: game.tiles.length, total: runObj.level.tiles.length,
|
||||
pieces: pieceRoutes(game),
|
||||
cells: [...game.level.cells].sort((a, b) => a.r - b.r || a.q - b.q).map(c => ({ q: c.q, r: c.r, pairs: pairs(c.tracks) })),
|
||||
last: last ? { action: last.action, result: last.result } : null,
|
||||
};
|
||||
if (format === 'json') return state;
|
||||
const lines = [];
|
||||
lines.push(`LEVEL ${state.level} · ${state.cells.length} 格 · 剩余 ${state.remaining}/${state.total} 枚 · 朝向 ${state.orientation} · 已用 ${state.step}/${state.budget} 步 · 状态 ${state.status}`);
|
||||
if (state.last) lines.push(`上一步：${describeAction(state.last.action)} → ${describeResult(state.last.result)}`);
|
||||
lines.push('每枚棋子当前的路线（箭头出发 → 依次经过的格 → 出界/闭环；[tX] = 那格上压着棋子 tX）：');
|
||||
for (const p of state.pieces) lines.push(' ' + routeLine(p));
|
||||
if (format === 'full') {
|
||||
lines.push('格子轨道（q,r: 端口配对 a-b；端口 0=↑ 1=↗ 2=↘ 3=↓ 4=↙ 5=↖）：');
|
||||
let row = [];
|
||||
for (const c of state.cells) {
|
||||
row.push(`${c.q},${c.r}:${c.pairs.replace(/ /g, ',')}`);
|
||||
if (row.length === 4) { lines.push(' ' + row.join(' ')); row = []; }
|
||||
}
|
||||
if (row.length) lines.push(' ' + row.join(' '));
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function describeAction(action) {
|
||||
if (action.type === 'tap') return `点击 ${action.id}`;
|
||||
if (action.type === 'rotate') return action.direction === 1 ? '右转' : '左转';
|
||||
if (action.type === 'peek') return `查看 ${action.id}`;
|
||||
if (action.type === 'undo') return '撤销';
|
||||
if (action.type === 'giveup') return '放弃';
|
||||
return action.type;
|
||||
}
|
||||
export function describeResult(result) {
|
||||
if (!result) return '';
|
||||
if (result.status === 'exit') return `离场（走了 ${result.cells} 格、${result.bends} 个弯）${result.opened ? `，另有 ${result.opened} 枚棋子的路通了` : ''}`;
|
||||
if (result.status === 'blocked') return `被 ${result.blocker} 挡住，退回原位`;
|
||||
if (result.status === 'loop') return '路线绕回原处，没有出口';
|
||||
if (result.status === 'rotated') return `底盘转到朝向 ${result.orientation}${result.opened ? `，${result.opened} 枚棋子的出口打开了` : '，这个朝向没有新的出口'}`;
|
||||
if (result.status === 'undone') return '已撤销上一步';
|
||||
if (result.status === 'peek') return result.route === 'exit' ? `它现在能离场：走 ${result.cells} 格、${result.bends} 个弯，经过 ${result.path}` : result.route === 'blocked' ? `它现在会在 ${result.path} 被 ${result.blocker} 挡住` : '它的路线绕回原处';
|
||||
if (result.status === 'cleared') return '全部离场，通关';
|
||||
if (result.status === 'rejected') return `无效：${result.reason}`;
|
||||
return result.status;
|
||||
}
|
||||
|
||||
export function act(runObj, action) {
|
||||
if (runObj.status !== 'playing') return reject(runObj, action, `本局已结束（${runObj.status}）`);
|
||||
if (runObj.steps.length >= runObj.budget) { runObj.status = 'budget'; return reject(runObj, action, '步数预算用完'); }
|
||||
if (!ACTIONS.includes(action?.type)) return reject(runObj, action, `未知动作 ${action?.type}，可用：${ACTIONS.join(' / ')}`);
|
||||
if (action.type === 'peek' && !runObj.peek) return reject(runObj, action, '本局未开启 peek');
|
||||
const game = replay(runObj);
|
||||
let result;
|
||||
if (action.type === 'giveup') {
|
||||
runObj.status = 'gave-up'; runObj.reason = action.reason || '';
|
||||
result = { status: 'gave-up', reason: runObj.reason };
|
||||
} else if (action.type === 'undo') {
|
||||
result = game.undo() ? { status: 'undone' } : { status: 'rejected', reason: '没有可撤销的步骤' };
|
||||
if (result.status === 'rejected') return reject(runObj, action, result.reason);
|
||||
} else if (action.type === 'rotate') {
|
||||
if (![1, -1].includes(action.direction)) return reject(runObj, action, 'rotate 需要 direction 1（右）或 -1（左）');
|
||||
const before = new Set(available(game.level, game.tiles).map(t => t.id));
|
||||
const turned = game.rotate(action.direction);
|
||||
if (turned.status !== 'rotated') return reject(runObj, action, turned.status);
|
||||
const opened = available(game.level, game.tiles).filter(t => !before.has(t.id)).length;
|
||||
result = { status: 'rotated', orientation: game.orientation, opened };
|
||||
} else {
|
||||
const tile = game.tiles.find(t => t.id === action.id);
|
||||
if (!tile) return reject(runObj, action, `没有棋子 ${action.id}`);
|
||||
if (action.type === 'peek') {
|
||||
const route = trace(game.level, tile, game.tiles);
|
||||
const path = route.steps.map(s => `${s.q},${s.r}`).join('→');
|
||||
result = { status: 'peek', route: route.status, cells: route.steps.length, bends: countBends(route), blocker: route.blocker || null, path };
|
||||
} else {
|
||||
const before = new Set(available(game.level, game.tiles).map(t => t.id));
|
||||
const route = game.tap(tile.id);
|
||||
if (route.status === 'exit') {
|
||||
const opened = available(game.level, game.tiles).filter(t => !before.has(t.id)).length;
|
||||
result = { status: 'exit', cells: route.steps.length, bends: countBends(route), opened };
|
||||
} else result = { status: route.status, blocker: route.blocker || null };
|
||||
}
|
||||
}
|
||||
// Wall-clock stamp per step. Steps in one batch land microseconds apart, so
|
||||
// the gap between batches is the agent's real decision latency.
|
||||
const step = { n: runObj.steps.length + 1, action, note: action.note || '', result, remaining: game.tiles.length, orientation: game.orientation, batch: null, at: Date.now() };
|
||||
runObj.steps.push(step);
|
||||
if (game.won) { runObj.status = 'cleared'; step.result = { ...result, cleared: true }; }
|
||||
else if (runObj.steps.length >= runObj.budget && runObj.status === 'playing') runObj.status = 'budget';
|
||||
return { step, feedback: describeResult(step.result) + (game.won ? '。全部离场，通关。' : '') };
|
||||
}
|
||||
function reject(runObj, action, reason) {
|
||||
return { step: null, feedback: `无效：${reason}`, rejected: { action, reason } };
|
||||
}
|
||||
|
||||
// Several actions in one decision, the way a person chains the obvious taps.
|
||||
// The chain stops at the first surprise (a tap that does not leave) and after
|
||||
// any turn — the board has to be re-read after either.
|
||||
export function actMany(runObj, actions, { note = '' } = {}) {
|
||||
const last = runObj.steps[runObj.steps.length - 1];
|
||||
const batch = (last?.batch || 0) + 1;
|
||||
const outcomes = [];
|
||||
let stopped = null;
|
||||
for (let i = 0; i < actions.length; i++) {
|
||||
const action = { ...actions[i], note: actions[i].note || note };
|
||||
const out = act(runObj, action);
|
||||
if (out.step) out.step.batch = batch;
|
||||
outcomes.push(out);
|
||||
if (!out.step || runObj.status !== 'playing') break;
|
||||
if (action.type === 'tap' && out.step.result.status !== 'exit' && i < actions.length - 1) { stopped = '这一步没成功，后面的动作没有执行'; break; }
|
||||
if (action.type === 'rotate' && i < actions.length - 1) { stopped = '旋转后盘面已变，后面的动作没有执行'; break; }
|
||||
}
|
||||
return { outcomes, stopped, batch };
|
||||
}
|
||||
|
||||
// One decision = one look at the board: a lone step, or a whole chained batch.
|
||||
// Its latency is the wall-clock gap since the previous decision was committed,
|
||||
// which for a shell agent includes its own thinking and process startup.
|
||||
export function decisionTimeline(runObj) {
|
||||
const groups = [];
|
||||
for (const step of runObj.steps) {
|
||||
const previous = groups[groups.length - 1];
|
||||
if (previous && step.batch != null && previous.batch === step.batch) { previous.steps.push(step); previous.at = step.at ?? previous.at; }
|
||||
else groups.push({ batch: step.batch, steps: [step], at: step.at ?? null, from: step.n });
|
||||
}
|
||||
let cursor = runObj.startedAt ? Date.parse(runObj.startedAt) : null;
|
||||
return groups.map(group => {
|
||||
const ms = group.at != null && cursor != null ? group.at - cursor : null;
|
||||
if (group.at != null) cursor = group.at;
|
||||
return { from: group.from, actions: group.steps.length, at: group.at, ms };
|
||||
});
|
||||
}
|
||||
|
||||
export const RATING_QUESTIONS = [
|
||||
{ key: 'clarity', text: '路线看得清吗（1 完全看不清 – 5 一眼看清）' },
|
||||
{ key: 'rotation', text: '旋转有意义吗（1 随便转都行/没用 – 5 每次转都是关键决定）' },
|
||||
{ key: 'stuck', text: '有没有觉得无解的时刻（1 从未 – 5 长时间以为无解）' },
|
||||
{ key: 'fun', text: '好玩吗（1 – 5）' },
|
||||
{ key: 'difficulty', text: '难度（1 太简单 – 5 太难）' },
|
||||
{ key: 'worst', text: '最困惑的一步是第几步，为什么（文字）' },
|
||||
];
|
||||
export function rate(runObj, answers) {
|
||||
const rating = {};
|
||||
for (const q of RATING_QUESTIONS) rating[q.key] = answers?.[q.key] ?? null;
|
||||
runObj.rating = rating;
|
||||
return rating;
|
||||
}
|
||||
|
||||
// Post-hoc metrics. The solver is allowed here — the run is over.
|
||||
export function metrics(runObj) {
|
||||
const game = new Game(runObj.level);
|
||||
const solution = solveRotations(runObj.level);
|
||||
const m = { run: runObj.run, level: runObj.level.id, persona: runObj.persona, model: runObj.model, agent: runObj.agent || '', peek: runObj.peek, status: runObj.status,
|
||||
steps: runObj.steps.length, taps: 0, exits: 0, deadTaps: 0, grazeDeadTaps: 0, longestDeadStreak: 0, rotations: 0, effectiveRotations: 0, blindRotations: 0,
|
||||
wrongWayRotations: 0, longestBlindStreak: 0, undos: 0, peeks: 0, stuckEntries: 0, stepsToFirstExit: null,
|
||||
minimumRotations: solution.solvable ? solution.rotations : null, cleared: runObj.status === 'cleared', gaveUp: runObj.status === 'gave-up', rating: runObj.rating };
|
||||
let dead = 0, blind = 0;
|
||||
for (const step of runObj.steps) {
|
||||
const a = step.action;
|
||||
if (a.type === 'tap') {
|
||||
m.taps++;
|
||||
const tile = game.tiles.find(t => t.id === a.id);
|
||||
const geoCells = tile ? new Set(trace(game.level, tile, [], true).steps.map(s => `${s.q},${s.r}`)) : new Set();
|
||||
const route = game.tap(a.id);
|
||||
if (route.status === 'exit') { m.exits++; dead = 0; blind = 0; if (m.stepsToFirstExit === null) m.stepsToFirstExit = step.n; }
|
||||
else {
|
||||
m.deadTaps++; dead++; m.longestDeadStreak = Math.max(m.longestDeadStreak, dead);
|
||||
const blocker = route.blocker && game.tiles.find(t => t.id === route.blocker);
|
||||
if (blocker && !geoCells.has(`${blocker.q},${blocker.r}`)) m.grazeDeadTaps++;
|
||||
}
|
||||
} else if (a.type === 'rotate') {
|
||||
m.rotations++;
|
||||
const other = available(rotateBoard(game.level, -a.direction), game.tiles).length;
|
||||
const before = new Set(available(game.level, game.tiles).map(t => t.id));
|
||||
game.rotate(a.direction);
|
||||
const opened = available(game.level, game.tiles).filter(t => !before.has(t.id)).length;
|
||||
if (opened) { m.effectiveRotations++; blind = 0; }
|
||||
else { m.blindRotations++; blind++; m.longestBlindStreak = Math.max(m.longestBlindStreak, blind); if (other) m.wrongWayRotations++; }
|
||||
} else if (a.type === 'undo') { m.undos++; game.undo(); }
|
||||
else if (a.type === 'peek') m.peeks++;
|
||||
if (!game.won && a.type !== 'peek' && !available(game.level, game.tiles).length) m.stuckEntries++;
|
||||
}
|
||||
m.rotationEfficiency = m.rotations ? m.effectiveRotations / m.rotations : null;
|
||||
m.actionsPerPiece = runObj.level.tiles.length ? m.steps / runObj.level.tiles.length : null;
|
||||
const timeline = decisionTimeline(runObj);
|
||||
m.decisions = timeline.length;
|
||||
m.actionsPerDecision = timeline.length ? m.steps / timeline.length : null;
|
||||
const latencies = timeline.map(d => d.ms).filter(ms => typeof ms === 'number' && ms >= 0);
|
||||
const sorted = [...latencies].sort((a, b) => a - b);
|
||||
const at = p => sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] : null;
|
||||
const lastAt = runObj.steps.length ? runObj.steps[runObj.steps.length - 1].at : null;
|
||||
m.timing = {
|
||||
totalMs: lastAt != null && runObj.startedAt ? lastAt - Date.parse(runObj.startedAt) : null,
|
||||
firstDecisionMs: latencies.length ? latencies[0] : null,
|
||||
medianDecisionMs: at(.5), p90DecisionMs: at(.9), maxDecisionMs: sorted.length ? sorted[sorted.length - 1] : null,
|
||||
// Pure model latency, when the direct-API loop recorded it.
|
||||
thinkMs: runObj.timing?.decisions?.length ? runObj.timing.decisions.reduce((s, d) => s + d.thinkMs, 0) : null,
|
||||
};
|
||||
return m;
|
||||
}
|
||||
Reference in New Issue
Block a user