#!/usr/bin/env node
// Build docs/bench-reliability.html — what the agent bench can and cannot tell
// us about level quality, computed from bench/runs.
//
//   node scripts/build-bench-report.mjs [--out=docs/bench-reliability.html]
//
// Every number in the page comes from the run files; nothing is hand-written.
// Batches are identified by the `agent` field, so a run's provenance (which
// model, which board format) is never guessed.
import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { metrics } from '../src/bench.js';

const flags = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
const out = resolve(flags.out || 'docs/bench-reliability.html');
const RUNS = resolve('bench/runs');

// ---------- data ----------
const set = JSON.parse(readFileSync(resolve('levels/prototypes/d5.json'), 'utf8'));
const levels = Array.isArray(set) ? set : set.levels;
// Levels are keyed by id, never by position: regenerating the set renumbers
// everything, and keying by ordinal silently dropped every run played against
// an earlier set. Runs carry their own level, so ids seen only in the archive
// still resolve.
const describe = (l, i) => ({
  n: i == null ? null : i + 1,
  family: l.id.replace('proto-d5-', '').replace(/-r\d+-s\d+$/, ''),
  lock: l.prototype ? Math.round(l.prototype.stats.lockedShare * 100) : null,
  minRot: l.prototype ? l.prototype.stats.rotations : null,
  cells: l.cells.length, tiles: l.tiles.length,
});
const meta = Object.fromEntries(levels.map((l, i) => [l.id, describe(l, i)]));

const BATCHES = [
  { tag: 'Grok Build CLI · xhigh', label: 'grok-4.6 · 旧盘面', short: 'grok旧', format: 'old' },
  { tag: 'pi · Grok 4.6 xhigh', label: 'grok-4.6 · 新盘面 · 第 1 次', short: 'grok新①', format: 'new' },
  { tag: 'pi · Grok 4.6 xhigh · 复测', label: 'grok-4.6 · 新盘面 · 第 2 次', short: 'grok新②', format: 'new' },
  { tag: 'Codex CLI · xhigh', label: 'codex-spark · 旧盘面', short: 'spark旧', format: 'old' },
  { tag: 'Codex CLI · gpt-6-astra low', label: 'gpt-6-astra · low · 新盘面', short: 'astra', format: 'new' },
  { tag: 'Codex CLI · spark xhigh', label: 'codex-spark · xhigh · 新盘面', short: 'spark新', format: 'new' },
  { tag: 'Kimi CLI · k3 high', label: 'kimi-k3 · high · 新盘面', short: 'kimi', format: 'new' },
  { tag: 'pi · Grok 4.6 xhigh · 新门槛', label: 'grok-4.6 · 新关卡集（终局门槛后）', short: 'grok新关', format: 'new' },
];

// Batches run under the fixed board format, i.e. everything that can be
// compared across models without the format change confounding it.
const NEW_TAGS = ['pi · Grok 4.6 xhigh', 'pi · Grok 4.6 xhigh · 复测', 'Kimi CLI · k3 high',
  'Codex CLI · gpt-6-astra low', 'Codex CLI · spark xhigh'];

const runs = readdirSync(RUNS).filter(f => f.endsWith('.json') && f !== 'index.json')
  .map(f => { try { return JSON.parse(readFileSync(resolve(RUNS, f), 'utf8')); } catch { return null; } })
  .filter(Boolean).filter(d => d.level?.id?.startsWith('proto-d5-'));
// Levels retired by a regeneration still have runs worth comparing.
let retired = 0;
for (const d of runs) if (!meta[d.level.id]) { meta[d.level.id] = describe(d.level, null); retired++; }
// Give every level a stable ordinal: current set first, retired ones after.
let ord = levels.length;
for (const k of Object.keys(meta)) if (meta[k].n == null) meta[k].n = ++ord;

const byTag = {};
for (const b of BATCHES) {
  const rows = {};
  for (const d of runs) if (d.agent === b.tag) rows[d.level.id] = { m: metrics(d), lv: meta[d.level.id], run: d.run, id: d.level.id };
  byTag[b.tag] = rows;
}
const present = BATCHES.filter(b => Object.keys(byTag[b.tag]).length > 0);

// ---------- stats ----------
const mean = a => a.length ? a.reduce((x, y) => x + y, 0) / a.length : null;
const sd = a => { const m = mean(a); return a.length ? Math.sqrt(mean(a.map(x => (x - m) ** 2))) : null; };
const pearson = (u, v) => {
  const n = u.length; if (n < 3) return null;
  const mu = mean(u), mv = mean(v);
  let su = 0, sv = 0, suv = 0;
  for (let i = 0; i < n; i++) { const a = u[i] - mu, c = v[i] - mv; su += a * a; sv += c * c; suv += a * c; }
  return (su && sv) ? suv / Math.sqrt(su * sv) : null;
};

const METRICS = [
  { key: 'extraRot', label: '多转', kind: 'mech', get: m => m.rotations - m.minimumRotations, hint: '清盘旋转数 − 求解器最少旋转数' },
  { key: 'blindRotations', label: '盲转', kind: 'mech', get: m => m.blindRotations, hint: '转了一次，什么出口都没打开' },
  { key: 'wrongWayRotations', label: '转错方向', kind: 'mech', get: m => m.wrongWayRotations, hint: '另一边本来能打开' },
  { key: 'steps', label: '步数', kind: 'mech', get: m => m.steps },
  { key: 'decisions', label: '决策数', kind: 'mech', get: m => m.decisions, hint: '看了几次盘面' },
  { key: 'deadTaps', label: '空点', kind: 'mech', get: m => m.deadTaps, hint: '点了没动' },
  { key: 'actionsPerPiece', label: '动作÷棋子', kind: 'mech', get: m => m.actionsPerPiece },
  { key: 'r.stuck', label: '评分：无解感', kind: 'rating', get: m => Number(m.rating?.stuck) },
  { key: 'r.difficulty', label: '评分：难度', kind: 'rating', get: m => Number(m.rating?.difficulty) },
  { key: 'r.rotation', label: '评分：旋转意义', kind: 'rating', get: m => Number(m.rating?.rotation) },
  { key: 'r.clarity', label: '评分：清晰度', kind: 'rating', get: m => Number(m.rating?.clarity) },
  { key: 'r.fun', label: '评分：好玩', kind: 'rating', get: m => Number(m.rating?.fun) },
];

function pair(tagA, tagB) {
  const A = byTag[tagA] || {}, B = byTag[tagB] || {};
  const shared = Object.keys(A).filter(k => B[k]).sort((x, y) => meta[x].n - meta[y].n);
  return { shared, A, B };
}

function reliability(tagA, tagB) {
  const { shared, A, B } = pair(tagA, tagB);
  if (shared.length < 3) return null;
  return METRICS.map(def => {
    // Drop only the levels a metric is missing on, not the whole metric: a run
    // still in progress has no rating yet, and losing every rating column to
    // one unfinished level would hide exactly what this table is testing.
    const ok = shared.filter(L => {
      const a = def.get(A[L].m), b = def.get(B[L].m);
      return a != null && b != null && !Number.isNaN(a) && !Number.isNaN(b);
    });
    if (ok.length < 3) return null;
    const u = ok.map(L => def.get(A[L].m)), v = ok.map(L => def.get(B[L].m));
    const r = pearson(u, v);
    const spread = sd(u);
    const same = u.filter((x, i) => x === v[i]).length;
    const n = ok.length;
    // A metric is usable for ranking levels only if it both separates levels
    // and survives a repeat run.
    const verdict = spread < 0.35 ? 'flat' : (r == null || r < 0.5) ? 'noise' : r >= 0.9 ? 'solid' : 'ok';
    return { ...def, r, spread, same, n, verdict, u, v };
  }).filter(Boolean);
}

// ---------- html helpers ----------
const esc = s => String(s).replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
const f2 = x => x == null ? '—' : (Math.round(x * 100) / 100).toFixed(2);
const f1 = x => x == null ? '—' : (Math.round(x * 10) / 10).toFixed(1);
const secs = ms => ms == null ? '—' : ms >= 10000 ? `${Math.round(ms / 1000)} s` : `${(ms / 1000).toFixed(1)} s`;

const VERDICT = {
  solid: ['可用 · 高', 'good'], ok: ['可用', 'good'],
  noise: ['噪声', 'signal'], flat: ['无区分度', 'warn'],
};

// Scatter of one metric across two runs of the same conditions.
function scatter(m, size = 168) {
  const pad = 26, span = size - pad - 10;
  const all = [...m.u, ...m.v];
  const lo = Math.min(...all), hi = Math.max(...all);
  const range = (hi - lo) || 1;
  const X = x => pad + ((x - lo) / range) * span;
  const Y = y => size - pad - ((y - lo) / range) * span;
  const dots = m.u.map((x, i) => `<circle cx="${X(x).toFixed(1)}" cy="${Y(m.v[i]).toFixed(1)}" r="3.6"/>`).join('');
  return `<svg viewBox="0 0 ${size} ${size}" role="img" aria-label="${esc(m.label)} 两次运行散点图，相关系数 ${f2(m.r)}">
  <line class="ax" x1="${pad}" y1="${size - pad}" x2="${size - 8}" y2="${size - pad}"/>
  <line class="ax" x1="${pad}" y1="8" x2="${pad}" y2="${size - pad}"/>
  <line class="diag" x1="${X(lo)}" y1="${Y(lo)}" x2="${X(hi)}" y2="${Y(hi)}"/>
  <g class="dot ${m.verdict}">${dots}</g>
  <text class="axl" x="${size / 2}" y="${size - 6}" text-anchor="middle">第 1 次</text>
  <text class="axl" x="10" y="${size / 2}" text-anchor="middle" transform="rotate(-90 10 ${size / 2})">第 2 次</text>
</svg>`;
}

// Per-family bar of a mechanical metric.
function familyBars(tag, key) {
  const rows = Object.values(byTag[tag]);
  if (!rows.length) return '';
  const fams = {};
  for (const r of rows) (fams[r.lv.family] ||= []).push(key === 'extraRot' ? r.m.rotations - r.m.minimumRotations : r.m[key]);
  const entries = Object.entries(fams).map(([k, v]) => ({ k, avg: mean(v), n: v.length })).sort((a, b) => b.avg - a.avg);
  const max = Math.max(...entries.map(e => e.avg), 1);
  return `<div class="bars">${entries.map(e => `
    <div class="bar-row"><span class="bar-k">${esc(e.k)}<small>${e.n} 关</small></span>
    <span class="bar-track"><i style="width:${(e.avg / max * 100).toFixed(1)}%" class="${e.avg === 0 ? 'zero' : ''}"></i></span>
    <b class="bar-v">${f1(e.avg)}</b></div>`).join('')}</div>`;
}

function batchSummary(b) {
  const rows = Object.values(byTag[b.tag]);
  if (!rows.length) return null;
  const S = f => rows.reduce((a, r) => a + f(r.m), 0);
  const A = f => mean(rows.map(r => f(r.m)));
  return {
    ...b, n: rows.length, cleared: rows.filter(r => r.m.cleared).length,
    dead: S(m => m.deadTaps), graze: S(m => m.grazeDeadTaps),
    blind: S(m => m.blindRotations), wrong: S(m => m.wrongWayRotations),
    extra: S(m => m.rotations - m.minimumRotations),
    app: A(m => m.actionsPerPiece), apd: A(m => m.actionsPerDecision),
    total: A(m => m.timing.totalMs), dec: A(m => m.timing.medianDecisionMs),
    ratings: Object.fromEntries(['clarity', 'rotation', 'stuck', 'fun', 'difficulty']
      .map(k => [k, mean(rows.map(r => Number(r.m.rating?.[k])).filter(x => !Number.isNaN(x)))])),
  };
}

// ---------- assemble ----------
const summaries = present.map(batchSummary).filter(Boolean);
const clean = reliability('pi · Grok 4.6 xhigh', 'pi · Grok 4.6 xhigh · 复测');
const confounded = reliability('Grok Build CLI · xhigh', 'pi · Grok 4.6 xhigh');

// worst-step notes, grouped by the theme they name
const THEMES = [
  { key: 'graze', label: '擦碰看不出来', re: /擦|紧贴/ },
  { key: 'turn', label: '不知道该转哪边', re: /转哪|该不该|哪个方向|左转还是右转|转错|继续转/ },
  { key: 'long', label: '长路线读不动', re: /太长|\d{2}\s*格|看不出|分不清/ },
];
function notes() {
  const outp = [];
  for (const b of present) for (const r of Object.values(byTag[b.tag])) {
    const w = r.m.rating?.worst; if (!w) continue;
    outp.push({ batch: b.short, L: r.lv.n, fam: r.lv.family, text: String(w) });
  }
  return outp.sort((a, b) => a.L - b.L);
}
const allNotes = notes();
const themed = THEMES.map(t => ({ ...t, hits: allNotes.filter(n => t.re.test(n.text)) }));

const allCleared = summaries.every(s => s.cleared === s.n);
const anyRunning = summaries.some(s => s.cleared < s.n);
const clearRateNote = allCleared
  ? '<p class="sub">所有批次通关率都是 100%。<b>这本身就是一条结论：通关率在这批内容上毫无区分度，任何流程都不该建立在通过/失败上。</b></p>'
  : `<p class="sub">已完成的对局通关率全部是 100%${anyRunning ? '（个别批次仍有对局在进行中，未计入）' : ''}。<b>通关率在这批内容上毫无区分度，任何流程都不该建立在通过/失败上。</b></p>`;

// Agreement on the rotation triple across every batch that saw the fixed board.
const liveTags = NEW_TAGS.filter(t => Object.keys(byTag[t] || {}).length);
const triple = m => [m.rotations - m.minimumRotations, m.blindRotations, m.wrongWayRotations].join('/');
const crossRows = [];
const crossIds = [...new Set(liveTags.flatMap(t => Object.keys(byTag[t] || {})))]
  .filter(id => liveTags.every(t => byTag[t]?.[id]))
  .sort((a, b) => meta[a].n - meta[b].n);
for (const id of crossIds) {
  const cells = liveTags.map(t => triple(byTag[t][id].m));
  crossRows.push({ L: meta[id].n, id, family: meta[id].family, cells, agree: new Set(cells).size === 1 });
}
const agreeN = crossRows.filter(r => r.agree).length;
const crossModel = crossRows.length < 3 ? '<p class="sub">同盘面批次不足，跨模型比较尚未成立。</p>' : `
<p>下表是同一套盘面下所有批次的 <code>多转 / 盲转 / 转错方向</code>。不同厂商的模型、不同 harness、各自独立跑——如果这些数字真的由关卡决定，它们就该一样。</p>
<div class="grid">
  <div class="tile"><div class="k">逐关完全一致</div><div class="v">${agreeN} / ${crossRows.length}</div><div class="n">三项数字一个不差</div></div>
  <div class="tile"><div class="k">参与批次</div><div class="v">${liveTags.length}</div><div class="n">${liveTags.map(t => BATCHES.find(b => b.tag === t).short).join(' · ')}</div></div>
</div>
<div class="scroll"><table>
<thead><tr><th>关</th><th>家族</th>${liveTags.map(t => `<th class="n">${esc(BATCHES.find(b => b.tag === t).short)}</th>`).join('')}<th>一致</th></tr></thead>
<tbody>
${crossRows.map(r => `<tr><td>L${r.L}</td><td>${esc(r.family)}</td>${r.cells.map(c => `<td class="n">${c}</td>`).join('')}<td>${r.agree ? '<span class="tag good">✔</span>' : '<span class="tag signal">✘</span>'}</td></tr>`).join('\n')}
</tbody></table></div>
<div class="callout good">
<p><b>这是全份报告里最硬的一条结论。</b>${agreeN}/${crossRows.length} 关上，独立运行的不同模型给出了完全相同的旋转三元组。这意味着<b>规则机器人 12 秒扫出来的这三个数，等价于大模型跑几小时的结果</b>——这个维度完全不需要 agent。</p>
</div>
${crossRows.filter(r => !r.agree).length ? `
<h3>不一致的关：不是难，是不稳定</h3>
<p class="sub">下面这些关在不同运行之间给出了不同的数字。排除模型差异后，剩下的解释是<b>同一关的体验取决于起手方向</b>——两个玩家玩同一关，一个觉得还行，一个觉得这关有毛病。这比"稳定地难"更值得处理。</p>
${crossRows.filter(r => !r.agree).map(r => `<div class="note"><b>L${r.L} ${esc(r.family)}</b>：${r.cells.join(' ／ ')}</div>`).join('')}` : ''}`;

const generated = new Date().toLocaleString('sv-SE', { timeZoneName: 'short' }).slice(0, 16)
  + ' ' + new Date().toLocaleTimeString('en-US', { timeZoneName: 'short' }).split(' ').pop();

const html = `<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>关卡实验台：信度检验与 1000 关流程</title>
<style>
:root{
  --ground:#EEF1F4; --surface:#FFFFFF; --inset:#E3E8ED; --panel:#F7F9FA;
  --ink:#141E28; --ink2:#48596A; --ink3:#6B7A88;
  --rule:#D3DAE1; --rule2:#BFC9D2;
  --rail:#435F78; --rail-soft:#8FA6BA;
  --signal:#B23A26; --signal-soft:#F2DDD8;
  --warn:#8A5C0F; --warn-soft:#F4E9D4;
  --good:#236E64; --good-soft:#D9EDEA;
  --shadow:0 1px 2px rgba(20,30,40,.06),0 6px 18px rgba(20,30,40,.05);
  color-scheme:light;
}
@media (prefers-color-scheme:dark){:root:not([data-theme=light]){
  --ground:#0F161C; --surface:#141D25; --inset:#0A1015; --panel:#141D25;
  --ink:#E4EAEF; --ink2:#AEBDC9; --ink3:#8494A2;
  --rule:#28343F; --rule2:#35434F;
  --rail:#8CB0CC; --rail-soft:#4E6577;
  --signal:#E4836C; --signal-soft:#3A211B;
  --warn:#E0AE5E; --warn-soft:#33280F;
  --good:#4FC7B7; --good-soft:#11302C;
  --shadow:0 1px 2px rgba(0,0,0,.3),0 6px 18px rgba(0,0,0,.28);
  color-scheme:dark;
}}
*{box-sizing:border-box}
body{margin:0;background:var(--ground);color:var(--ink);
  font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei",ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;
  line-height:1.72;-webkit-font-smoothing:antialiased}
.wrap{max-width:1000px;margin:0 auto;padding:0 24px 88px}
code,.mono{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-variant-numeric:tabular-nums}
header.head{padding:60px 0 30px;border-bottom:1px solid var(--rule);margin-bottom:40px}
.eyebrow{font-size:11.5px;letter-spacing:.18em;text-transform:uppercase;color:var(--ink3);margin-bottom:14px}
h1{font-size:clamp(28px,4.4vw,42px);line-height:1.18;margin:0 0 16px;letter-spacing:-.02em;text-wrap:balance}
.lede{font-size:17px;color:var(--ink2);max-width:66ch;margin:0}
.meta{margin-top:20px;font-size:12.5px;color:var(--ink3)}
h2{font-size:23px;margin:56px 0 8px;letter-spacing:-.01em;text-wrap:balance}
h2 .num{color:var(--rail-soft);font-family:ui-monospace,monospace;font-size:15px;margin-right:10px}
h3{font-size:16px;margin:32px 0 10px;color:var(--ink)}
p{margin:0 0 14px;max-width:72ch}
.sub{color:var(--ink2);font-size:14.5px}
.card{background:var(--surface);border:1px solid var(--rule);border-radius:12px;padding:20px 22px;box-shadow:var(--shadow);margin:20px 0}
.callout{border-left:3px solid var(--rail);background:var(--panel);border-radius:0 10px 10px 0;padding:16px 20px;margin:20px 0}
.callout.bad{border-left-color:var(--signal);background:var(--signal-soft)}
.callout.good{border-left-color:var(--good);background:var(--good-soft)}
.callout p:last-child{margin-bottom:0}
.callout b{font-weight:650}
table{width:100%;border-collapse:collapse;font-size:13.5px;margin:16px 0}
th,td{text-align:left;padding:9px 10px;border-bottom:1px solid var(--rule)}
th{font-size:11.5px;letter-spacing:.06em;color:var(--ink3);text-transform:uppercase;font-weight:600;white-space:nowrap}
td.n,th.n{text-align:right;font-family:ui-monospace,monospace;font-variant-numeric:tabular-nums;white-space:nowrap}
tbody tr:hover{background:var(--panel)}
.scroll{overflow-x:auto;-webkit-overflow-scrolling:touch}
.tag{display:inline-block;font-size:11px;padding:2px 8px;border-radius:20px;font-weight:600;white-space:nowrap}
.tag.good{background:var(--good-soft);color:var(--good)}
.tag.signal{background:var(--signal-soft);color:var(--signal)}
.tag.warn{background:var(--warn-soft);color:var(--warn)}
.grid{display:grid;gap:16px;grid-template-columns:repeat(auto-fit,minmax(210px,1fr));margin:20px 0}
.tile{background:var(--surface);border:1px solid var(--rule);border-radius:12px;padding:16px 18px}
.tile .k{font-size:11.5px;letter-spacing:.06em;color:var(--ink3);text-transform:uppercase}
.tile .v{font-family:ui-monospace,monospace;font-size:27px;font-weight:600;line-height:1.25;margin-top:4px;font-variant-numeric:tabular-nums}
.tile .n{font-size:12.5px;color:var(--ink2);margin-top:2px}
.scatters{display:grid;gap:14px;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));margin:18px 0}
.sc{background:var(--surface);border:1px solid var(--rule);border-radius:12px;padding:12px 12px 6px;text-align:center}
.sc h4{margin:0 0 2px;font-size:13.5px;font-weight:600}
.sc .r{font-family:ui-monospace,monospace;font-size:12.5px;color:var(--ink2)}
.sc svg{width:100%;height:auto;display:block;margin-top:4px}
.ax{stroke:var(--rule2);stroke-width:1}
.diag{stroke:var(--rail-soft);stroke-width:1;stroke-dasharray:3 3}
.axl{font-size:9px;fill:var(--ink3);font-family:ui-monospace,monospace}
.dot circle{fill:var(--rail);fill-opacity:.72}
.dot.solid circle{fill:var(--good);fill-opacity:.8}
.dot.noise circle{fill:var(--signal);fill-opacity:.7}
.dot.flat circle{fill:var(--warn);fill-opacity:.7}
.bars{margin:14px 0}
.bar-row{display:grid;grid-template-columns:140px 1fr 48px;align-items:center;gap:12px;padding:5px 0}
.bar-k{font-size:13px}
.bar-k small{color:var(--ink3);margin-left:6px;font-size:11px}
.bar-track{background:var(--inset);border-radius:4px;height:9px;overflow:hidden}
.bar-track i{display:block;height:100%;background:var(--signal);border-radius:4px}
.bar-track i.zero{background:var(--good)}
.bar-v{font-family:ui-monospace,monospace;font-size:13px;text-align:right}
.note{border-left:2px solid var(--rule2);padding:2px 0 2px 14px;margin:10px 0;font-size:13.5px;color:var(--ink2)}
.note b{color:var(--ink);font-weight:600}
.steps{counter-reset:s;list-style:none;padding:0;margin:22px 0}
.steps>li{counter-increment:s;position:relative;padding:0 0 22px 46px;border-left:1px solid var(--rule);margin-left:14px}
.steps>li:last-child{border-left-color:transparent;padding-bottom:0}
.steps>li::before{content:"T" counter(s);position:absolute;left:-15px;top:0;width:30px;height:30px;border-radius:50%;
  background:var(--rail);color:var(--surface);display:grid;place-items:center;font-family:ui-monospace,monospace;font-size:11px;font-weight:700}
.steps h3{margin:2px 0 6px}
footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:var(--ink3);font-size:12.5px}
@media(max-width:640px){.bar-row{grid-template-columns:100px 1fr 42px}.wrap{padding:0 16px 60px}}
</style>
</head>
<body>
<div class="wrap">

<header class="head">
  <div class="eyebrow">HEX TRAILS · 关卡实验台</div>
  <h1>Agent 评级里，哪些数字真能拿来做关卡</h1>
  <p class="lede">同一批关卡、同一个模型、同一套盘面，跑两遍。能重现的指标才配进流程——这份报告把每个指标逐个检验了一遍，结论和直觉相反。</p>
  <p class="meta mono">生成于 ${generated} · 数据来自 bench/runs 的 ${runs.length} 局 · 关卡集 levels/prototypes/d5.json（18 关）</p>
</header>

<h2><span class="num">01</span>参与检验的批次</h2>
<p class="sub">每一批用 <code>agent</code> 字段标识，不靠文件名猜来源。「旧盘面」指擦碰只报第一个阻挡者的版本，「新盘面」指修复后全部列出的版本。</p>
<p class="sub">模型名以 <b>CLI 启动时打印的解析结果</b>为准，而不是提示词里让 agent 自报的名字——有一批 18 局曾被标成 codex-spark，实际是配置在运行前被改掉、跑成了 gpt-6-astra/low，存档已按日志头更正。</p>
<div class="scroll"><table>
<thead><tr><th>批次</th><th class="n">关数</th><th class="n">通关</th><th class="n">空点</th><th class="n">擦碰空点</th><th class="n">盲转</th><th class="n">转错</th><th class="n">多转</th><th class="n">单局均</th><th class="n">每决策</th></tr></thead>
<tbody>
${summaries.map(s => `<tr><td>${esc(s.label)}</td><td class="n">${s.n}</td><td class="n">${s.cleared}/${s.n}</td><td class="n">${s.dead}</td><td class="n">${s.graze}</td><td class="n">${s.blind}</td><td class="n">${s.wrong}</td><td class="n">${s.extra}</td><td class="n">${secs(s.total)}</td><td class="n">${secs(s.dec)}</td></tr>`).join('\n')}
</tbody></table></div>
${clearRateNote}

${clean ? `
<h2><span class="num">02</span>信度检验：跑两遍还一样吗</h2>
<p>同模型（grok-4.6 xhigh）、同 harness（pi）、同盘面（新）、同 ${clean[0].n} 关，只是又跑了一遍。两次之间唯一的变量就是运行本身，所以下面的相关系数<b>就是这个指标的复测信度</b>。</p>
<p class="sub">判定规则：<b>关间标准差 &lt; 0.35</b> 说明这个指标在不同关卡上几乎一样，再稳也没用（无法排序）；<b>复测 r &lt; 0.5</b> 说明换一次运行结果就变，不能作为依据。两条都过才算可用。</p>
<div class="scroll"><table>
<thead><tr><th>指标</th><th class="n">关间标准差</th><th class="n">复测相关 r</th><th class="n">逐关完全一致</th><th>判定</th></tr></thead>
<tbody>
${clean.map(m => `<tr><td>${esc(m.label)}${m.hint ? `<br><small class="sub">${esc(m.hint)}</small>` : ''}</td><td class="n">${f2(m.spread)}</td><td class="n">${f2(m.r)}</td><td class="n">${m.same}/${m.n}</td><td><span class="tag ${VERDICT[m.verdict][1]}">${VERDICT[m.verdict][0]}</span></td></tr>`).join('\n')}
</tbody></table></div>
<p class="sub">「关间标准差」衡量这个指标能不能把不同关卡区分开——标准差接近 0 的指标就算完全可复现也没用，因为每关都一样。<b>两个条件要同时满足</b>：既能区分关卡，又能跑两遍不变。</p>

<div class="scatters">
${clean.filter(m => m.r != null).slice(0, 8).map(m => `<div class="sc"><h4>${esc(m.label)}</h4><div class="r">r = ${f2(m.r)}</div>${scatter(m)}</div>`).join('\n')}
</div>
<p class="sub">虚线是「两次完全相同」。点贴着虚线 = 可复现；点散开 = 这个数字换一次运行就变。</p>
` : '<h2><span class="num">02</span>信度检验</h2><p class="sub">复测批次尚未完成。</p>'}

${confounded ? `
<h3>对照：换了盘面格式的那一对</h3>
<p class="sub">上一节控制了所有变量。作为对照，这里是<b>旧盘面 vs 新盘面</b>的同 18 关配对——它多了一个变量（盘面变了），所以相关系数是信度的<b>下限</b>而非真值。两张表一起看，能分清「指标本身不稳」和「盘面变化导致的差异」。</p>
<div class="scroll"><table>
<thead><tr><th>指标</th><th class="n">控制变量后 r</th><th class="n">换盘面后 r</th><th>差异说明</th></tr></thead>
<tbody>
${(clean || []).map(m => {
  const c = confounded.find(x => x.key === m.key);
  if (!c) return '';
  const delta = (m.r != null && c.r != null) ? m.r - c.r : null;
  const note = delta == null ? '—' : Math.abs(delta) < 0.15 ? '两者接近，盘面变化影响很小'
    : delta > 0 ? '控制变量后明显更高——之前被盘面变化压低了' : '控制变量后反而更低——值得留意';
  return `<tr><td>${esc(m.label)}</td><td class="n">${f2(m.r)}</td><td class="n">${f2(c.r)}</td><td class="sub">${note}</td></tr>`;
}).join('\n')}
</tbody></table></div>
` : ''}

<h2><span class="num">03</span>跨模型交叉验证：这些数字属于关卡，不属于玩家</h2>
${crossModel}

<h2><span class="num">04</span>家族结构：机械指标指向同一个地方</h2>
<p>把「多转」按路线家族平均。这是关卡结构问题，不是玩家问题——两批 grok、以及 spark，在这一点上给出一致的结果。</p>
${present.filter(b => Object.keys(byTag[b.tag]).length >= 6).map(b => `
<div class="card"><h3 style="margin-top:0">${esc(b.label)} · 各家族平均多转</h3>${familyBars(b.tag, 'extraRot')}</div>`).join('\n')}
<div class="callout bad">
<p><b>orbits 和 spiral 是问题家族。</b>它们稳定地需要 7 次旋转（最少 3 次）外加 4 次盲转——等于把 6 个朝向挨个试一遍。而 petals 和 crossing 是干净的 0。锁住率解释不了这个差异（L16 也是 orbits、锁 78%，却是 3/3）。</p>
</div>

<h2><span class="num">05</span>Agent 说了什么</h2>
<p class="sub">评分是 1–5 的数字，<code>worst</code> 是它自己写的一句话。下面按反复出现的主题归类——<b>价值在重复模式，不在单条内容</b>。</p>
${themed.filter(t => t.hits.length).map(t => `
<h3>${esc(t.label)} · ${t.hits.length} 次提及</h3>
${t.hits.slice(0, 4).map(n => `<div class="note"><b>L${n.L} ${esc(n.fam)}</b>（${esc(n.batch)}）：${esc(n.text.slice(0, 110))}</div>`).join('\n')}`).join('\n')}

<h2><span class="num">06</span>1000 关该怎么跑</h2>
<p>核心判断：<b>机械计数器当主力，agent 做抽检</b>。免费的规则机器人能扫全量并稳定给出盲转/多转/转错；agent 贵但能提供两样机器给不了的东西——难度的相对排序，和用自然语言点出的系统性缺陷。<b>关键是只收集经过信度检验的那几项</b>。</p>
<ol class="steps">
<li><h3>生成期硬门槛 · 零成本</h3><p class="sub">用已有求解器，在生成时就卡掉：最少旋转 ≥ 2、开局锁住率 ≤ 90%、首转两个方向都能推进、无死局状态。不合格的根本不进候选池。</p></li>
<li><h3>规则机器人全量扫 · 1000 关 12 秒</h3><p class="sub">产出盲转 / 多转 / 转错方向。按 <code>盲转 ≥ 2 或 多转 ≥ 2</code> 打标。这条规则在 D5 上正好命中 orbits+spiral 那几关，和两个大模型跑两小时得到的结论一致。</p></li>
<li><h3>Agent 只跑被标记的 + 5% 随机抽样</h3><p class="sub">不是 1000 关全跑。收集<b>三样东西</b>：<code>difficulty</code>（r=0.93，可用来给 1000 关排序）、<code>stuck</code>（r=0.85，体感无解的直接信号）、以及 <code>worst</code> 文字（重复模式能挖出系统性缺陷）。<b>丢掉 rotation / clarity / fun</b>——复测 r 只有 0.32 / 0.19 / 0.56，是模型噪声。按实测 ${summaries.length ? secs(mean(summaries.filter(s => s.format === 'new').map(s => s.total))) : '—'}/关、4 路并行，150 关约半小时。</p></li>
<li><h3>真人标定锚点 · 5–10 关</h3><p class="sub">Agent 的难度<b>排序</b>可信（复测 r=0.93），但绝对值可能整体偏移——它觉得「刚好」的关，真人可能觉得太简单。所以真人只需要玩少数几关当<b>锚点</b>，把 agent 的相对排序映射到真实难度刻度上，不必逐关评。这比原本设想的省很多人力。</p></li>
</ol>
<div class="callout good">
<p><b>最该做的一件事：把它变成回归测试，而不是评审流程。</b>改完生成器，12 秒跑一遍规则机器人，比较盲转分布。从 22% 降到 5% 就是客观改善，升上去就是刚引入了退化。12 秒一轮可以进 CI；几小时一轮且分数不可复现的人工评审，不可能成为迭代循环。</p>
</div>

<h2><span class="num">07</span>这份结论的边界</h2>
<p class="sub">写清楚哪里不够硬，比多列几个数字重要。</p>
<ul class="sub">
<li>跨模型一致性只验证了 <b>6 关</b>（spark 与 grok 的交集），其中 1 关不一致。机械指标"与玩家无关"这个说法，样本还太小。</li>
<li>复测只做了 <b>2 次</b>。2 次能算相关系数，但对评分这种低方差的离散量，2 次的估计本身就有不小的不确定性。</li>
<li>Agent 读的是文字不是像素，<b>测不出"看不清"</b>。clarity 这一项从原理上就不该指望它。</li>
<li>所有结论都基于 <b>D5 原型这 18 关</b>。这批关卡通关率 100%、难度评分方差极小，换一批更难的内容，结论可能不同。</li>
</ul>

<footer>
<p class="mono">数据：bench/runs · 生成脚本：scripts/build-bench-report.mjs · 重新生成：<code>node scripts/build-bench-report.mjs</code></p>
<p>本页所有数字由脚本从存档直接计算，未经手工填写。相关系数为 Pearson r，标准差为总体标准差。</p>
</footer>

</div>
</body>
</html>`;

writeFileSync(out, html);
const kb = (Buffer.byteLength(html) / 1024).toFixed(0);
console.log(`报告已生成：${out}（${kb} KB）`);
console.log(`  批次 ${summaries.length} 个 · 对局 ${runs.length} 局`);
if (clean) console.log(`  干净信度检验：${clean[0].n} 关配对 · ${clean.filter(m => m.verdict === 'solid' || m.verdict === 'ok').length}/${clean.length} 个指标判定可用`);
else console.log('  ⚠ 复测批次数据不足，信度一节留空');