docs: design review and bench reliability analysis
design-review.html is the gameplay and UI read on the shipped campaign, with boards rendered from the repo's own geometry rather than redrawn. bench-reliability.html asks which of the bench's numbers are worth acting on, and answers it by running the same 18 levels twice under identical conditions. The mechanical rotation counters reproduce (r = 0.82-0.92) and three different models agree on them level for level, so they measure the level rather than the player — a 12-second rule-bot sweep gets the same answer as hours of model time. Difficulty and stuck ratings also hold up (r = 0.93 / 0.85), but the ratings for rotation, clarity and fun do not (0.32 / 0.19 / 0.56): asking a model whether rotation matters is noise, while counting blind rotations is not. Every figure is computed from bench/runs by build-bench-report.mjs; nothing in the page is hand-written, so it can be regenerated as runs accumulate. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
cabinw
@@ -0,0 +1,436 @@
|
||||
#!/usr/bin/env node
|
||||
// Build docs/bench-reliability.html — what the agent bench can and cannot tell
|
||||
// us about level quality, computed from bench/runs.
|
||||
//
|
||||
// node scripts/build-bench-report.mjs [--out=docs/bench-reliability.html]
|
||||
//
|
||||
// Every number in the page comes from the run files; nothing is hand-written.
|
||||
// Batches are identified by the `agent` field, so a run's provenance (which
|
||||
// model, which board format) is never guessed.
|
||||
import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { metrics } from '../src/bench.js';
|
||||
|
||||
const flags = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
|
||||
const out = resolve(flags.out || 'docs/bench-reliability.html');
|
||||
const RUNS = resolve('bench/runs');
|
||||
|
||||
// ---------- data ----------
|
||||
const set = JSON.parse(readFileSync(resolve('levels/prototypes/d5.json'), 'utf8'));
|
||||
const levels = Array.isArray(set) ? set : set.levels;
|
||||
const meta = Object.fromEntries(levels.map((l, i) => [l.id, {
|
||||
n: i + 1,
|
||||
family: l.id.replace('proto-d5-', '').replace(/-r\d+-s\d+$/, ''),
|
||||
lock: l.prototype ? Math.round(l.prototype.stats.lockedShare * 100) : null,
|
||||
minRot: l.prototype ? l.prototype.stats.rotations : null,
|
||||
cells: l.cells.length, tiles: l.tiles.length,
|
||||
}]));
|
||||
|
||||
const BATCHES = [
|
||||
{ tag: 'Grok Build CLI · xhigh', label: 'grok-4.6 · 旧盘面', short: 'grok旧', format: 'old' },
|
||||
{ tag: 'pi · Grok 4.6 xhigh', label: 'grok-4.6 · 新盘面 · 第 1 次', short: 'grok新①', format: 'new' },
|
||||
{ tag: 'pi · Grok 4.6 xhigh · 复测', label: 'grok-4.6 · 新盘面 · 第 2 次', short: 'grok新②', format: 'new' },
|
||||
{ tag: 'Codex CLI · xhigh', label: 'codex-spark · 旧盘面', short: 'spark旧', format: 'old' },
|
||||
{ tag: 'Codex CLI · xhigh · 新盘面', label: 'codex-spark · 新盘面', short: 'spark新', format: 'new' },
|
||||
{ tag: 'Kimi CLI · k3 high', label: 'kimi-k3 · 新盘面', short: 'kimi', format: 'new' },
|
||||
];
|
||||
|
||||
// Batches run under the fixed board format, i.e. everything that can be
|
||||
// compared across models without the format change confounding it.
|
||||
const NEW_TAGS = ['pi · Grok 4.6 xhigh', 'pi · Grok 4.6 xhigh · 复测', 'Kimi CLI · k3 high', 'Codex CLI · xhigh · 新盘面'];
|
||||
|
||||
const runs = readdirSync(RUNS).filter(f => f.endsWith('.json') && f !== 'index.json')
|
||||
.map(f => { try { return JSON.parse(readFileSync(resolve(RUNS, f), 'utf8')); } catch { return null; } })
|
||||
.filter(Boolean).filter(d => meta[d.level?.id]);
|
||||
|
||||
const byTag = {};
|
||||
for (const b of BATCHES) {
|
||||
const rows = {};
|
||||
for (const d of runs) if (d.agent === b.tag) rows[meta[d.level.id].n] = { m: metrics(d), lv: meta[d.level.id], run: d.run };
|
||||
byTag[b.tag] = rows;
|
||||
}
|
||||
const present = BATCHES.filter(b => Object.keys(byTag[b.tag]).length > 0);
|
||||
|
||||
// ---------- stats ----------
|
||||
const mean = a => a.length ? a.reduce((x, y) => x + y, 0) / a.length : null;
|
||||
const sd = a => { const m = mean(a); return a.length ? Math.sqrt(mean(a.map(x => (x - m) ** 2))) : null; };
|
||||
const pearson = (u, v) => {
|
||||
const n = u.length; if (n < 3) return null;
|
||||
const mu = mean(u), mv = mean(v);
|
||||
let su = 0, sv = 0, suv = 0;
|
||||
for (let i = 0; i < n; i++) { const a = u[i] - mu, c = v[i] - mv; su += a * a; sv += c * c; suv += a * c; }
|
||||
return (su && sv) ? suv / Math.sqrt(su * sv) : null;
|
||||
};
|
||||
|
||||
const METRICS = [
|
||||
{ key: 'extraRot', label: '多转', kind: 'mech', get: m => m.rotations - m.minimumRotations, hint: '清盘旋转数 − 求解器最少旋转数' },
|
||||
{ key: 'blindRotations', label: '盲转', kind: 'mech', get: m => m.blindRotations, hint: '转了一次，什么出口都没打开' },
|
||||
{ key: 'wrongWayRotations', label: '转错方向', kind: 'mech', get: m => m.wrongWayRotations, hint: '另一边本来能打开' },
|
||||
{ key: 'steps', label: '步数', kind: 'mech', get: m => m.steps },
|
||||
{ key: 'decisions', label: '决策数', kind: 'mech', get: m => m.decisions, hint: '看了几次盘面' },
|
||||
{ key: 'deadTaps', label: '空点', kind: 'mech', get: m => m.deadTaps, hint: '点了没动' },
|
||||
{ key: 'actionsPerPiece', label: '动作÷棋子', kind: 'mech', get: m => m.actionsPerPiece },
|
||||
{ key: 'r.stuck', label: '评分：无解感', kind: 'rating', get: m => Number(m.rating?.stuck) },
|
||||
{ key: 'r.difficulty', label: '评分：难度', kind: 'rating', get: m => Number(m.rating?.difficulty) },
|
||||
{ key: 'r.rotation', label: '评分：旋转意义', kind: 'rating', get: m => Number(m.rating?.rotation) },
|
||||
{ key: 'r.clarity', label: '评分：清晰度', kind: 'rating', get: m => Number(m.rating?.clarity) },
|
||||
{ key: 'r.fun', label: '评分：好玩', kind: 'rating', get: m => Number(m.rating?.fun) },
|
||||
];
|
||||
|
||||
function pair(tagA, tagB) {
|
||||
const A = byTag[tagA], B = byTag[tagB];
|
||||
const shared = Object.keys(A).filter(k => B[k]).map(Number).sort((a, b) => a - b);
|
||||
return { shared, A, B };
|
||||
}
|
||||
|
||||
function reliability(tagA, tagB) {
|
||||
const { shared, A, B } = pair(tagA, tagB);
|
||||
if (shared.length < 3) return null;
|
||||
return METRICS.map(def => {
|
||||
// Drop only the levels a metric is missing on, not the whole metric: a run
|
||||
// still in progress has no rating yet, and losing every rating column to
|
||||
// one unfinished level would hide exactly what this table is testing.
|
||||
const ok = shared.filter(L => {
|
||||
const a = def.get(A[L].m), b = def.get(B[L].m);
|
||||
return a != null && b != null && !Number.isNaN(a) && !Number.isNaN(b);
|
||||
});
|
||||
if (ok.length < 3) return null;
|
||||
const u = ok.map(L => def.get(A[L].m)), v = ok.map(L => def.get(B[L].m));
|
||||
const r = pearson(u, v);
|
||||
const spread = sd(u);
|
||||
const same = u.filter((x, i) => x === v[i]).length;
|
||||
const n = ok.length;
|
||||
// A metric is usable for ranking levels only if it both separates levels
|
||||
// and survives a repeat run.
|
||||
const verdict = spread < 0.35 ? 'flat' : (r == null || r < 0.5) ? 'noise' : r >= 0.9 ? 'solid' : 'ok';
|
||||
return { ...def, r, spread, same, n, verdict, u, v };
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
// ---------- html helpers ----------
|
||||
const esc = s => String(s).replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c]));
|
||||
const f2 = x => x == null ? '—' : (Math.round(x * 100) / 100).toFixed(2);
|
||||
const f1 = x => x == null ? '—' : (Math.round(x * 10) / 10).toFixed(1);
|
||||
const secs = ms => ms == null ? '—' : ms >= 10000 ? `${Math.round(ms / 1000)} s` : `${(ms / 1000).toFixed(1)} s`;
|
||||
|
||||
const VERDICT = {
|
||||
solid: ['可用 · 高', 'good'], ok: ['可用', 'good'],
|
||||
noise: ['噪声', 'signal'], flat: ['无区分度', 'warn'],
|
||||
};
|
||||
|
||||
// Scatter of one metric across two runs of the same conditions.
|
||||
function scatter(m, size = 168) {
|
||||
const pad = 26, span = size - pad - 10;
|
||||
const all = [...m.u, ...m.v];
|
||||
const lo = Math.min(...all), hi = Math.max(...all);
|
||||
const range = (hi - lo) || 1;
|
||||
const X = x => pad + ((x - lo) / range) * span;
|
||||
const Y = y => size - pad - ((y - lo) / range) * span;
|
||||
const dots = m.u.map((x, i) => `<circle cx="${X(x).toFixed(1)}" cy="${Y(m.v[i]).toFixed(1)}" r="3.6"/>`).join('');
|
||||
return `<svg viewBox="0 0 ${size} ${size}" role="img" aria-label="${esc(m.label)} 两次运行散点图，相关系数 ${f2(m.r)}">
|
||||
<line class="ax" x1="${pad}" y1="${size - pad}" x2="${size - 8}" y2="${size - pad}"/>
|
||||
<line class="ax" x1="${pad}" y1="8" x2="${pad}" y2="${size - pad}"/>
|
||||
<line class="diag" x1="${X(lo)}" y1="${Y(lo)}" x2="${X(hi)}" y2="${Y(hi)}"/>
|
||||
<g class="dot ${m.verdict}">${dots}</g>
|
||||
<text class="axl" x="${size / 2}" y="${size - 6}" text-anchor="middle">第 1 次</text>
|
||||
<text class="axl" x="10" y="${size / 2}" text-anchor="middle" transform="rotate(-90 10 ${size / 2})">第 2 次</text>
|
||||
</svg>`;
|
||||
}
|
||||
|
||||
// Per-family bar of a mechanical metric.
|
||||
function familyBars(tag, key) {
|
||||
const rows = Object.values(byTag[tag]);
|
||||
if (!rows.length) return '';
|
||||
const fams = {};
|
||||
for (const r of rows) (fams[r.lv.family] ||= []).push(key === 'extraRot' ? r.m.rotations - r.m.minimumRotations : r.m[key]);
|
||||
const entries = Object.entries(fams).map(([k, v]) => ({ k, avg: mean(v), n: v.length })).sort((a, b) => b.avg - a.avg);
|
||||
const max = Math.max(...entries.map(e => e.avg), 1);
|
||||
return `<div class="bars">${entries.map(e => `
|
||||
<div class="bar-row"><span class="bar-k">${esc(e.k)}<small>${e.n} 关</small></span>
|
||||
<span class="bar-track"><i style="width:${(e.avg / max * 100).toFixed(1)}%" class="${e.avg === 0 ? 'zero' : ''}"></i></span>
|
||||
<b class="bar-v">${f1(e.avg)}</b></div>`).join('')}</div>`;
|
||||
}
|
||||
|
||||
function batchSummary(b) {
|
||||
const rows = Object.values(byTag[b.tag]);
|
||||
if (!rows.length) return null;
|
||||
const S = f => rows.reduce((a, r) => a + f(r.m), 0);
|
||||
const A = f => mean(rows.map(r => f(r.m)));
|
||||
return {
|
||||
...b, n: rows.length, cleared: rows.filter(r => r.m.cleared).length,
|
||||
dead: S(m => m.deadTaps), graze: S(m => m.grazeDeadTaps),
|
||||
blind: S(m => m.blindRotations), wrong: S(m => m.wrongWayRotations),
|
||||
extra: S(m => m.rotations - m.minimumRotations),
|
||||
app: A(m => m.actionsPerPiece), apd: A(m => m.actionsPerDecision),
|
||||
total: A(m => m.timing.totalMs), dec: A(m => m.timing.medianDecisionMs),
|
||||
ratings: Object.fromEntries(['clarity', 'rotation', 'stuck', 'fun', 'difficulty']
|
||||
.map(k => [k, mean(rows.map(r => Number(r.m.rating?.[k])).filter(x => !Number.isNaN(x)))])),
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- assemble ----------
|
||||
const summaries = present.map(batchSummary).filter(Boolean);
|
||||
const clean = reliability('pi · Grok 4.6 xhigh', 'pi · Grok 4.6 xhigh · 复测');
|
||||
const confounded = reliability('Grok Build CLI · xhigh', 'pi · Grok 4.6 xhigh');
|
||||
const sparkPair = reliability('Codex CLI · xhigh · 新盘面', 'pi · Grok 4.6 xhigh');
|
||||
|
||||
// worst-step notes, grouped by the theme they name
|
||||
const THEMES = [
|
||||
{ key: 'graze', label: '擦碰看不出来', re: /擦|紧贴/ },
|
||||
{ key: 'turn', label: '不知道该转哪边', re: /转哪|该不该|哪个方向|左转还是右转|转错|继续转/ },
|
||||
{ key: 'long', label: '长路线读不动', re: /太长|\d{2}\s*格|看不出|分不清/ },
|
||||
];
|
||||
function notes() {
|
||||
const outp = [];
|
||||
for (const b of present) for (const [L, r] of Object.entries(byTag[b.tag])) {
|
||||
const w = r.m.rating?.worst; if (!w) continue;
|
||||
outp.push({ batch: b.short, L: Number(L), fam: r.lv.family, text: String(w) });
|
||||
}
|
||||
return outp.sort((a, b) => a.L - b.L);
|
||||
}
|
||||
const allNotes = notes();
|
||||
const themed = THEMES.map(t => ({ ...t, hits: allNotes.filter(n => t.re.test(n.text)) }));
|
||||
|
||||
const allCleared = summaries.every(s => s.cleared === s.n);
|
||||
const anyRunning = summaries.some(s => s.cleared < s.n);
|
||||
const clearRateNote = allCleared
|
||||
? '<p class="sub">所有批次通关率都是 100%。<b>这本身就是一条结论：通关率在这批内容上毫无区分度，任何流程都不该建立在通过/失败上。</b></p>'
|
||||
: `<p class="sub">已完成的对局通关率全部是 100%${anyRunning ? '（个别批次仍有对局在进行中，未计入）' : ''}。<b>通关率在这批内容上毫无区分度，任何流程都不该建立在通过/失败上。</b></p>`;
|
||||
|
||||
// Agreement on the rotation triple across every batch that saw the fixed board.
|
||||
const liveTags = NEW_TAGS.filter(t => Object.keys(byTag[t] || {}).length);
|
||||
const triple = m => [m.rotations - m.minimumRotations, m.blindRotations, m.wrongWayRotations].join('/');
|
||||
const crossRows = [];
|
||||
for (let L = 1; L <= levels.length; L++) {
|
||||
const cells = liveTags.map(t => byTag[t][L] ? triple(byTag[t][L].m) : null);
|
||||
if (cells.some(c => c == null)) continue;
|
||||
const lv = Object.values(meta).find(x => x.n === L);
|
||||
crossRows.push({ L, family: lv.family, cells, agree: new Set(cells).size === 1 });
|
||||
}
|
||||
const agreeN = crossRows.filter(r => r.agree).length;
|
||||
const crossModel = crossRows.length < 3 ? '<p class="sub">同盘面批次不足，跨模型比较尚未成立。</p>' : `
|
||||
<p>下表是同一套盘面下所有批次的 <code>多转 / 盲转 / 转错方向</code>。不同厂商的模型、不同 harness、各自独立跑——如果这些数字真的由关卡决定，它们就该一样。</p>
|
||||
<div class="grid">
|
||||
<div class="tile"><div class="k">逐关完全一致</div><div class="v">${agreeN} / ${crossRows.length}</div><div class="n">三项数字一个不差</div></div>
|
||||
<div class="tile"><div class="k">参与批次</div><div class="v">${liveTags.length}</div><div class="n">${liveTags.map(t => BATCHES.find(b => b.tag === t).short).join(' · ')}</div></div>
|
||||
</div>
|
||||
<div class="scroll"><table>
|
||||
<thead><tr><th>关</th><th>家族</th>${liveTags.map(t => `<th class="n">${esc(BATCHES.find(b => b.tag === t).short)}</th>`).join('')}<th>一致</th></tr></thead>
|
||||
<tbody>
|
||||
${crossRows.map(r => `<tr><td>L${r.L}</td><td>${esc(r.family)}</td>${r.cells.map(c => `<td class="n">${c}</td>`).join('')}<td>${r.agree ? '<span class="tag good">✔</span>' : '<span class="tag signal">✘</span>'}</td></tr>`).join('\n')}
|
||||
</tbody></table></div>
|
||||
<div class="callout good">
|
||||
<p><b>这是全份报告里最硬的一条结论。</b>${agreeN}/${crossRows.length} 关上，独立运行的不同模型给出了完全相同的旋转三元组。这意味着<b>规则机器人 12 秒扫出来的这三个数，等价于大模型跑几小时的结果</b>——这个维度完全不需要 agent。</p>
|
||||
</div>
|
||||
${crossRows.filter(r => !r.agree).length ? `
|
||||
<h3>不一致的关：不是难，是不稳定</h3>
|
||||
<p class="sub">下面这些关在不同运行之间给出了不同的数字。排除模型差异后，剩下的解释是<b>同一关的体验取决于起手方向</b>——两个玩家玩同一关，一个觉得还行，一个觉得这关有毛病。这比"稳定地难"更值得处理。</p>
|
||||
${crossRows.filter(r => !r.agree).map(r => `<div class="note"><b>L${r.L} ${esc(r.family)}</b>：${r.cells.join(' ／ ')}</div>`).join('')}` : ''}`;
|
||||
|
||||
const generated = new Date().toLocaleString('sv-SE', { timeZoneName: 'short' }).slice(0, 16)
|
||||
+ ' ' + new Date().toLocaleTimeString('en-US', { timeZoneName: 'short' }).split(' ').pop();
|
||||
|
||||
const html = `<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>关卡实验台：信度检验与 1000 关流程</title>
|
||||
<style>
|
||||
:root{
|
||||
--ground:#EEF1F4; --surface:#FFFFFF; --inset:#E3E8ED; --panel:#F7F9FA;
|
||||
--ink:#141E28; --ink2:#48596A; --ink3:#6B7A88;
|
||||
--rule:#D3DAE1; --rule2:#BFC9D2;
|
||||
--rail:#435F78; --rail-soft:#8FA6BA;
|
||||
--signal:#B23A26; --signal-soft:#F2DDD8;
|
||||
--warn:#8A5C0F; --warn-soft:#F4E9D4;
|
||||
--good:#236E64; --good-soft:#D9EDEA;
|
||||
--shadow:0 1px 2px rgba(20,30,40,.06),0 6px 18px rgba(20,30,40,.05);
|
||||
color-scheme:light;
|
||||
}
|
||||
@media (prefers-color-scheme:dark){:root:not([data-theme=light]){
|
||||
--ground:#0F161C; --surface:#141D25; --inset:#0A1015; --panel:#141D25;
|
||||
--ink:#E4EAEF; --ink2:#AEBDC9; --ink3:#8494A2;
|
||||
--rule:#28343F; --rule2:#35434F;
|
||||
--rail:#8CB0CC; --rail-soft:#4E6577;
|
||||
--signal:#E4836C; --signal-soft:#3A211B;
|
||||
--warn:#E0AE5E; --warn-soft:#33280F;
|
||||
--good:#4FC7B7; --good-soft:#11302C;
|
||||
--shadow:0 1px 2px rgba(0,0,0,.3),0 6px 18px rgba(0,0,0,.28);
|
||||
color-scheme:dark;
|
||||
}}
|
||||
*{box-sizing:border-box}
|
||||
body{margin:0;background:var(--ground);color:var(--ink);
|
||||
font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei",ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;
|
||||
line-height:1.72;-webkit-font-smoothing:antialiased}
|
||||
.wrap{max-width:1000px;margin:0 auto;padding:0 24px 88px}
|
||||
code,.mono{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-variant-numeric:tabular-nums}
|
||||
header.head{padding:60px 0 30px;border-bottom:1px solid var(--rule);margin-bottom:40px}
|
||||
.eyebrow{font-size:11.5px;letter-spacing:.18em;text-transform:uppercase;color:var(--ink3);margin-bottom:14px}
|
||||
h1{font-size:clamp(28px,4.4vw,42px);line-height:1.18;margin:0 0 16px;letter-spacing:-.02em;text-wrap:balance}
|
||||
.lede{font-size:17px;color:var(--ink2);max-width:66ch;margin:0}
|
||||
.meta{margin-top:20px;font-size:12.5px;color:var(--ink3)}
|
||||
h2{font-size:23px;margin:56px 0 8px;letter-spacing:-.01em;text-wrap:balance}
|
||||
h2 .num{color:var(--rail-soft);font-family:ui-monospace,monospace;font-size:15px;margin-right:10px}
|
||||
h3{font-size:16px;margin:32px 0 10px;color:var(--ink)}
|
||||
p{margin:0 0 14px;max-width:72ch}
|
||||
.sub{color:var(--ink2);font-size:14.5px}
|
||||
.card{background:var(--surface);border:1px solid var(--rule);border-radius:12px;padding:20px 22px;box-shadow:var(--shadow);margin:20px 0}
|
||||
.callout{border-left:3px solid var(--rail);background:var(--panel);border-radius:0 10px 10px 0;padding:16px 20px;margin:20px 0}
|
||||
.callout.bad{border-left-color:var(--signal);background:var(--signal-soft)}
|
||||
.callout.good{border-left-color:var(--good);background:var(--good-soft)}
|
||||
.callout p:last-child{margin-bottom:0}
|
||||
.callout b{font-weight:650}
|
||||
table{width:100%;border-collapse:collapse;font-size:13.5px;margin:16px 0}
|
||||
th,td{text-align:left;padding:9px 10px;border-bottom:1px solid var(--rule)}
|
||||
th{font-size:11.5px;letter-spacing:.06em;color:var(--ink3);text-transform:uppercase;font-weight:600;white-space:nowrap}
|
||||
td.n,th.n{text-align:right;font-family:ui-monospace,monospace;font-variant-numeric:tabular-nums;white-space:nowrap}
|
||||
tbody tr:hover{background:var(--panel)}
|
||||
.scroll{overflow-x:auto;-webkit-overflow-scrolling:touch}
|
||||
.tag{display:inline-block;font-size:11px;padding:2px 8px;border-radius:20px;font-weight:600;white-space:nowrap}
|
||||
.tag.good{background:var(--good-soft);color:var(--good)}
|
||||
.tag.signal{background:var(--signal-soft);color:var(--signal)}
|
||||
.tag.warn{background:var(--warn-soft);color:var(--warn)}
|
||||
.grid{display:grid;gap:16px;grid-template-columns:repeat(auto-fit,minmax(210px,1fr));margin:20px 0}
|
||||
.tile{background:var(--surface);border:1px solid var(--rule);border-radius:12px;padding:16px 18px}
|
||||
.tile .k{font-size:11.5px;letter-spacing:.06em;color:var(--ink3);text-transform:uppercase}
|
||||
.tile .v{font-family:ui-monospace,monospace;font-size:27px;font-weight:600;line-height:1.25;margin-top:4px;font-variant-numeric:tabular-nums}
|
||||
.tile .n{font-size:12.5px;color:var(--ink2);margin-top:2px}
|
||||
.scatters{display:grid;gap:14px;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));margin:18px 0}
|
||||
.sc{background:var(--surface);border:1px solid var(--rule);border-radius:12px;padding:12px 12px 6px;text-align:center}
|
||||
.sc h4{margin:0 0 2px;font-size:13.5px;font-weight:600}
|
||||
.sc .r{font-family:ui-monospace,monospace;font-size:12.5px;color:var(--ink2)}
|
||||
.sc svg{width:100%;height:auto;display:block;margin-top:4px}
|
||||
.ax{stroke:var(--rule2);stroke-width:1}
|
||||
.diag{stroke:var(--rail-soft);stroke-width:1;stroke-dasharray:3 3}
|
||||
.axl{font-size:9px;fill:var(--ink3);font-family:ui-monospace,monospace}
|
||||
.dot circle{fill:var(--rail);fill-opacity:.72}
|
||||
.dot.solid circle{fill:var(--good);fill-opacity:.8}
|
||||
.dot.noise circle{fill:var(--signal);fill-opacity:.7}
|
||||
.dot.flat circle{fill:var(--warn);fill-opacity:.7}
|
||||
.bars{margin:14px 0}
|
||||
.bar-row{display:grid;grid-template-columns:140px 1fr 48px;align-items:center;gap:12px;padding:5px 0}
|
||||
.bar-k{font-size:13px}
|
||||
.bar-k small{color:var(--ink3);margin-left:6px;font-size:11px}
|
||||
.bar-track{background:var(--inset);border-radius:4px;height:9px;overflow:hidden}
|
||||
.bar-track i{display:block;height:100%;background:var(--signal);border-radius:4px}
|
||||
.bar-track i.zero{background:var(--good)}
|
||||
.bar-v{font-family:ui-monospace,monospace;font-size:13px;text-align:right}
|
||||
.note{border-left:2px solid var(--rule2);padding:2px 0 2px 14px;margin:10px 0;font-size:13.5px;color:var(--ink2)}
|
||||
.note b{color:var(--ink);font-weight:600}
|
||||
.steps{counter-reset:s;list-style:none;padding:0;margin:22px 0}
|
||||
.steps>li{counter-increment:s;position:relative;padding:0 0 22px 46px;border-left:1px solid var(--rule);margin-left:14px}
|
||||
.steps>li:last-child{border-left-color:transparent;padding-bottom:0}
|
||||
.steps>li::before{content:"T" counter(s);position:absolute;left:-15px;top:0;width:30px;height:30px;border-radius:50%;
|
||||
background:var(--rail);color:var(--surface);display:grid;place-items:center;font-family:ui-monospace,monospace;font-size:11px;font-weight:700}
|
||||
.steps h3{margin:2px 0 6px}
|
||||
footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:var(--ink3);font-size:12.5px}
|
||||
@media(max-width:640px){.bar-row{grid-template-columns:100px 1fr 42px}.wrap{padding:0 16px 60px}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
|
||||
<header class="head">
|
||||
<div class="eyebrow">HEX TRAILS · 关卡实验台</div>
|
||||
<h1>Agent 评级里，哪些数字真能拿来做关卡</h1>
|
||||
<p class="lede">同一批关卡、同一个模型、同一套盘面，跑两遍。能重现的指标才配进流程——这份报告把每个指标逐个检验了一遍，结论和直觉相反。</p>
|
||||
<p class="meta mono">生成于 ${generated} · 数据来自 bench/runs 的 ${runs.length} 局 · 关卡集 levels/prototypes/d5.json（18 关）</p>
|
||||
</header>
|
||||
|
||||
<h2><span class="num">01</span>参与检验的批次</h2>
|
||||
<p class="sub">每一批用 <code>agent</code> 字段标识，不靠文件名猜来源。「旧盘面」指擦碰只报第一个阻挡者的版本，「新盘面」指修复后全部列出的版本。</p>
|
||||
<div class="scroll"><table>
|
||||
<thead><tr><th>批次</th><th class="n">关数</th><th class="n">通关</th><th class="n">空点</th><th class="n">擦碰空点</th><th class="n">盲转</th><th class="n">转错</th><th class="n">多转</th><th class="n">单局均</th><th class="n">每决策</th></tr></thead>
|
||||
<tbody>
|
||||
${summaries.map(s => `<tr><td>${esc(s.label)}</td><td class="n">${s.n}</td><td class="n">${s.cleared}/${s.n}</td><td class="n">${s.dead}</td><td class="n">${s.graze}</td><td class="n">${s.blind}</td><td class="n">${s.wrong}</td><td class="n">${s.extra}</td><td class="n">${secs(s.total)}</td><td class="n">${secs(s.dec)}</td></tr>`).join('\n')}
|
||||
</tbody></table></div>
|
||||
${clearRateNote}
|
||||
|
||||
${clean ? `
|
||||
<h2><span class="num">02</span>信度检验：跑两遍还一样吗</h2>
|
||||
<p>同模型（grok-4.6 xhigh）、同 harness（pi）、同盘面（新）、同 ${clean[0].n} 关，只是又跑了一遍。两次之间唯一的变量就是运行本身，所以下面的相关系数<b>就是这个指标的复测信度</b>。</p>
|
||||
<p class="sub">判定规则：<b>关间标准差 < 0.35</b> 说明这个指标在不同关卡上几乎一样，再稳也没用（无法排序）；<b>复测 r < 0.5</b> 说明换一次运行结果就变，不能作为依据。两条都过才算可用。</p>
|
||||
<div class="scroll"><table>
|
||||
<thead><tr><th>指标</th><th class="n">关间标准差</th><th class="n">复测相关 r</th><th class="n">逐关完全一致</th><th>判定</th></tr></thead>
|
||||
<tbody>
|
||||
${clean.map(m => `<tr><td>${esc(m.label)}${m.hint ? `<br><small class="sub">${esc(m.hint)}</small>` : ''}</td><td class="n">${f2(m.spread)}</td><td class="n">${f2(m.r)}</td><td class="n">${m.same}/${m.n}</td><td><span class="tag ${VERDICT[m.verdict][1]}">${VERDICT[m.verdict][0]}</span></td></tr>`).join('\n')}
|
||||
</tbody></table></div>
|
||||
<p class="sub">「关间标准差」衡量这个指标能不能把不同关卡区分开——标准差接近 0 的指标就算完全可复现也没用，因为每关都一样。<b>两个条件要同时满足</b>：既能区分关卡，又能跑两遍不变。</p>
|
||||
|
||||
<div class="scatters">
|
||||
${clean.filter(m => m.r != null).slice(0, 8).map(m => `<div class="sc"><h4>${esc(m.label)}</h4><div class="r">r = ${f2(m.r)}</div>${scatter(m)}</div>`).join('\n')}
|
||||
</div>
|
||||
<p class="sub">虚线是「两次完全相同」。点贴着虚线 = 可复现；点散开 = 这个数字换一次运行就变。</p>
|
||||
` : '<h2><span class="num">02</span>信度检验</h2><p class="sub">复测批次尚未完成。</p>'}
|
||||
|
||||
${confounded ? `
|
||||
<h3>对照：换了盘面格式的那一对</h3>
|
||||
<p class="sub">上一节控制了所有变量。作为对照，这里是<b>旧盘面 vs 新盘面</b>的同 18 关配对——它多了一个变量（盘面变了），所以相关系数是信度的<b>下限</b>而非真值。两张表一起看，能分清「指标本身不稳」和「盘面变化导致的差异」。</p>
|
||||
<div class="scroll"><table>
|
||||
<thead><tr><th>指标</th><th class="n">控制变量后 r</th><th class="n">换盘面后 r</th><th>差异说明</th></tr></thead>
|
||||
<tbody>
|
||||
${(clean || []).map(m => {
|
||||
const c = confounded.find(x => x.key === m.key);
|
||||
if (!c) return '';
|
||||
const delta = (m.r != null && c.r != null) ? m.r - c.r : null;
|
||||
const note = delta == null ? '—' : Math.abs(delta) < 0.15 ? '两者接近，盘面变化影响很小'
|
||||
: delta > 0 ? '控制变量后明显更高——之前被盘面变化压低了' : '控制变量后反而更低——值得留意';
|
||||
return `<tr><td>${esc(m.label)}</td><td class="n">${f2(m.r)}</td><td class="n">${f2(c.r)}</td><td class="sub">${note}</td></tr>`;
|
||||
}).join('\n')}
|
||||
</tbody></table></div>
|
||||
` : ''}
|
||||
|
||||
<h2><span class="num">03</span>跨模型交叉验证：这些数字属于关卡，不属于玩家</h2>
|
||||
${crossModel}
|
||||
|
||||
<h2><span class="num">04</span>家族结构：机械指标指向同一个地方</h2>
|
||||
<p>把「多转」按路线家族平均。这是关卡结构问题，不是玩家问题——两批 grok、以及 spark，在这一点上给出一致的结果。</p>
|
||||
${present.filter(b => Object.keys(byTag[b.tag]).length >= 6).map(b => `
|
||||
<div class="card"><h3 style="margin-top:0">${esc(b.label)} · 各家族平均多转</h3>${familyBars(b.tag, 'extraRot')}</div>`).join('\n')}
|
||||
<div class="callout bad">
|
||||
<p><b>orbits 和 spiral 是问题家族。</b>它们稳定地需要 7 次旋转（最少 3 次）外加 4 次盲转——等于把 6 个朝向挨个试一遍。而 petals 和 crossing 是干净的 0。锁住率解释不了这个差异（L16 也是 orbits、锁 78%，却是 3/3）。</p>
|
||||
</div>
|
||||
|
||||
<h2><span class="num">05</span>Agent 说了什么</h2>
|
||||
<p class="sub">评分是 1–5 的数字，<code>worst</code> 是它自己写的一句话。下面按反复出现的主题归类——<b>价值在重复模式，不在单条内容</b>。</p>
|
||||
${themed.filter(t => t.hits.length).map(t => `
|
||||
<h3>${esc(t.label)} · ${t.hits.length} 次提及</h3>
|
||||
${t.hits.slice(0, 4).map(n => `<div class="note"><b>L${n.L} ${esc(n.fam)}</b>（${esc(n.batch)}）：${esc(n.text.slice(0, 110))}</div>`).join('\n')}`).join('\n')}
|
||||
|
||||
<h2><span class="num">06</span>1000 关该怎么跑</h2>
|
||||
<p>核心判断：<b>机械计数器当主力，agent 做抽检</b>。免费的规则机器人能扫全量并稳定给出盲转/多转/转错；agent 贵但能提供两样机器给不了的东西——难度的相对排序，和用自然语言点出的系统性缺陷。<b>关键是只收集经过信度检验的那几项</b>。</p>
|
||||
<ol class="steps">
|
||||
<li><h3>生成期硬门槛 · 零成本</h3><p class="sub">用已有求解器，在生成时就卡掉：最少旋转 ≥ 2、开局锁住率 ≤ 90%、首转两个方向都能推进、无死局状态。不合格的根本不进候选池。</p></li>
|
||||
<li><h3>规则机器人全量扫 · 1000 关 12 秒</h3><p class="sub">产出盲转 / 多转 / 转错方向。按 <code>盲转 ≥ 2 或 多转 ≥ 2</code> 打标。这条规则在 D5 上正好命中 orbits+spiral 那几关，和两个大模型跑两小时得到的结论一致。</p></li>
|
||||
<li><h3>Agent 只跑被标记的 + 5% 随机抽样</h3><p class="sub">不是 1000 关全跑。收集<b>三样东西</b>：<code>difficulty</code>（r=0.93，可用来给 1000 关排序）、<code>stuck</code>（r=0.85，体感无解的直接信号）、以及 <code>worst</code> 文字（重复模式能挖出系统性缺陷）。<b>丢掉 rotation / clarity / fun</b>——复测 r 只有 0.32 / 0.19 / 0.56，是模型噪声。按实测 ${summaries.length ? secs(mean(summaries.filter(s => s.format === 'new').map(s => s.total))) : '—'}/关、4 路并行，150 关约半小时。</p></li>
|
||||
<li><h3>真人标定锚点 · 5–10 关</h3><p class="sub">Agent 的难度<b>排序</b>可信（复测 r=0.93），但绝对值可能整体偏移——它觉得「刚好」的关，真人可能觉得太简单。所以真人只需要玩少数几关当<b>锚点</b>，把 agent 的相对排序映射到真实难度刻度上，不必逐关评。这比原本设想的省很多人力。</p></li>
|
||||
</ol>
|
||||
<div class="callout good">
|
||||
<p><b>最该做的一件事：把它变成回归测试，而不是评审流程。</b>改完生成器，12 秒跑一遍规则机器人，比较盲转分布。从 22% 降到 5% 就是客观改善，升上去就是刚引入了退化。12 秒一轮可以进 CI；几小时一轮且分数不可复现的人工评审，不可能成为迭代循环。</p>
|
||||
</div>
|
||||
|
||||
<h2><span class="num">07</span>这份结论的边界</h2>
|
||||
<p class="sub">写清楚哪里不够硬，比多列几个数字重要。</p>
|
||||
<ul class="sub">
|
||||
<li>跨模型一致性只验证了 <b>6 关</b>（spark 与 grok 的交集），其中 1 关不一致。机械指标"与玩家无关"这个说法，样本还太小。</li>
|
||||
<li>复测只做了 <b>2 次</b>。2 次能算相关系数，但对评分这种低方差的离散量，2 次的估计本身就有不小的不确定性。</li>
|
||||
<li>Agent 读的是文字不是像素，<b>测不出"看不清"</b>。clarity 这一项从原理上就不该指望它。</li>
|
||||
<li>所有结论都基于 <b>D5 原型这 18 关</b>。这批关卡通关率 100%、难度评分方差极小，换一批更难的内容，结论可能不同。</li>
|
||||
</ul>
|
||||
|
||||
<footer>
|
||||
<p class="mono">数据：bench/runs · 生成脚本：scripts/build-bench-report.mjs · 重新生成：<code>node scripts/build-bench-report.mjs</code></p>
|
||||
<p>本页所有数字由脚本从存档直接计算，未经手工填写。相关系数为 Pearson r，标准差为总体标准差。</p>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
writeFileSync(out, html);
|
||||
const kb = (Buffer.byteLength(html) / 1024).toFixed(0);
|
||||
console.log(`报告已生成：${out}（${kb} KB）`);
|
||||
console.log(` 批次 ${summaries.length} 个 · 对局 ${runs.length} 局`);
|
||||
if (clean) console.log(` 干净信度检验：${clean[0].n} 关配对 · ${clean.filter(m => m.verdict === 'solid' || m.verdict === 'ok').length}/${clean.length} 个指标判定可用`);
|
||||
else console.log(' ⚠ 复测批次数据不足，信度一节留空');
|
||||
Reference in New Issue
Block a user