fix(bench): rebuild each round's level set from its runs, not from a snapshot

The round comparison was about to be wrong. levels/campaign-50-r2.json was
copied from levels/campaign-50.json to preserve "the set round 2 played", but
select-50 writes that path in place and several selection experiments ran
between the round and the copy. The file kept 28 of the 50 boards round 2 had
actually played and 46 of round 3's — so the comparison was scoring one round
against another round's levels, and it read as round 3 regressing.

Runs store the whole board and its campaign index, so a round is its own record
and cannot drift. round-set.mjs rebuilds it, the report uses that, and the
snapshot file is gone rather than left lying around looking authoritative.

With the rounds correctly matched, round 3 is not a regression and not an
improvement: back band 2.17 against round 2's 2.39, on a rating that retests at
0.55, which is not enough to separate seventeen levels. The report says that
rather than picking whichever reading flatters the change.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

scripts/build-process-report.mjs

@@ -7,6 +7,7 @@ import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { trace } from '../src/core.js';
 import { metrics } from '../src/bench.js';
+import { roundSet } from './round-set.mjs';
 import { deepLevelKey, legibility } from '../src/deep-lock.js';
 
 const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
@@ -29,23 +30,30 @@ for (const f of existsSync(resolve('bench/runs')) ? readdirSync(resolve('bench/r
     runs.push({ raw: d, m: metrics(d) });
   } catch { /* torn read */ }
 }
-// Rounds are matched to their own level set by board, not by id: the reselect
-// between them reissued ids from the ordinal.
-const inSet = new Set(levels.map(deepLevelKey));
-const oldLevels = (() => { try { return JSON.parse(readFileSync(resolve('levels/campaign-50-r1.json'), 'utf8')).levels; } catch { return []; } })();
-const inOld = new Set(oldLevels.map(deepLevelKey));
-const round = (agent, set) => runs.filter(r => r.raw.agent === agent && r.raw.level?.tiles && set.has(deepLevelKey(r.raw.level)));
-const r1 = round('R1 · grok xhigh', inOld);
-const r2 = round('R2 · grok xhigh', inSet);
+// Each round is rebuilt from its own runs. A disk snapshot of "the set round N
+// played" cannot be trusted — select-50 writes campaign-50.json in place, so a
+// selection experiment between the round and the copy replaces it silently, and
+// one such copy kept only 28 of its round's 50 boards.
+const ROUND_IDS = ['R1', 'R2', 'R3', 'R4'];
+const sets = Object.fromEntries(ROUND_IDS.map(id => [id, roundSet(id)]).filter(([, rows]) => rows.length));
+const present = Object.keys(sets);
+const LATEST = present[present.length - 1] ?? 'R1';
+const PREV = present[present.length - 2] ?? null;
+const flat = id => (sets[id] ?? []).flatMap(e => e.runs.map(raw => ({ raw, m: metrics(raw) })));
+const r1 = PREV ? flat(PREV) : [];
+const r2 = flat(LATEST);
 const rated = r2.filter(r => r.m.rating && Number.isFinite(Number(r.m.rating.difficulty)));
 const byDiff = rated.reduce((m, r) => (m[r.m.rating.difficulty] = (m[r.m.rating.difficulty] || 0) + 1, m), {});
 const topDiff = Object.entries(byDiff).sort((a, b) => b[1] - a[1])[0] ?? ['—', 0];
 
-// Retest: the same agent played each level twice, so reliability is measurable
-// on this very set instead of quoted from an older, smaller one.
+// Retest pairs, wherever a round happened to play a level twice. Round 2 did it
+// across the board by accident, which is where the reliability figures come from.
 const pairsBy = new Map();
-for (const r of r2) { const k = deepLevelKey(r.raw.level); if (!pairsBy.has(k)) pairsBy.set(k, []); pairsBy.get(k).push(r); }
-const pairs = [...pairsBy.values()].filter(v => v.length >= 2).map(v => [v[0], v[1]]);
+for (const id of present) for (const e of sets[id]) {
+  const rows = e.runs.map(raw => ({ raw, m: metrics(raw) }));
+  if (rows.length >= 2) pairsBy.set(`${id}:${deepLevelKey(e.level)}`, rows);
+}
+const pairs = [...pairsBy.values()].map(v => [v[0], v[1]]);
 const sd = a => { const m = mean(a); return Math.sqrt(mean(a.map(v => (v - m) ** 2))); };
 const pearson = (A, B) => { const sa = sd(A), sb = sd(B); if (!sa || !sb) return null; const ma = mean(A), mb = mean(B); return mean(A.map((v, i) => (v - ma) * (B[i] - mb))) / (sa * sb); };
 const retest = get => {
@@ -58,6 +66,20 @@ const identical = pairs.filter(([a, b]) => sig(a) === sig(b)).length;
 const ratingDisagreed = pairs.filter(([a, b]) => a.m.rating?.difficulty != null && b.m.rating?.difficulty != null && a.m.rating.difficulty !== b.m.rating.difficulty);
 const disagreedOnSamePlay = ratingDisagreed.filter(([a, b]) => sig(a) === sig(b)).length;
 
+// A band is a slice of the round's own ordering, and each level counts once no
+// matter how many times it was played.
+const bandStats = (id, a, b) => {
+  const slice = (sets[id] ?? []).slice(a, b);
+  const per = get => slice.map(e => mean(e.runs.map(raw => get(metrics(raw))).filter(Number.isFinite))).filter(Number.isFinite);
+  return {
+    n: slice.length,
+    difficulty: mean(per(m => Number(m.rating?.difficulty))),
+    stuck: mean(per(m => Number(m.rating?.stuck))),
+    blind: mean(per(m => m.blindRotations ?? 0)),
+    contrast: mean(slice.map(e => e.level.campaign?.contrast ?? 0)),
+    rotations: mean(slice.map(e => e.level.campaign?.rotations ?? 0)),
+  };
+};
 const bandOf = (rows, a, b) => rows.filter(r => { const i = levels.findIndex(l => deepLevelKey(l) === deepLevelKey(r.raw.level)); return i >= a && i < b; });
 const BANDS = [['前档 L1–L12', 0, 12], ['中档 L13–L32', 12, 32], ['后档 L33–L50', 32, 50]];
 
@@ -172,9 +194,9 @@ a{color:var(--rail)}
 <div class="wrap">
 <h1>关卡制作流程复盘</h1>
 <p class="lede">这一轮做 50 关的过程里，最有用的结论是一个<b>否定结论</b>：原计划的「agent 打分 → 拟合权重 → 重选关卡」这个闭环<b>合不上</b>，而且不是样本量的问题。下面是它为什么合不上，以及流程该改成什么样。</p>
-<p class="meta">实测批次 R2 · grok-4.6 xhigh · ${R1.n} 局（50 关，多数跑了两次）· 对照批次 R1 ${R1.r1n} 局 · 生成于 ${new Date().toLocaleString('sv-SE').slice(0, 16)}</p>
+<p class="meta">最新批次 ${esc(LATEST)} · ${R1.n} 局 · 对照批次 ${esc(PREV ?? '—')} ${R1.r1n} 局 · 生成于 ${new Date().toLocaleString('sv-SE').slice(0, 16)}</p>
 
-<h2>第二轮实测到了什么</h2>
+<h2>最新一轮实测到了什么</h2>
 <div class="tiles">
   <div class="tile ${R1.cleared === R1.n ? 'ok' : 'miss'}"><div class="k">通关</div><div class="v">${R1.cleared}/${R1.n}</div><div class="n">无死局</div></div>
   <div class="tile ${R1.dead === 0 ? 'ok' : 'miss'}"><div class="k">空点 / 关</div><div class="v">${f2(R1.dead)}</div><div class="n">目标 0</div></div>
@@ -185,7 +207,7 @@ a{color:var(--rail)}
 <p>机械面干净，说明生成期的闸门是有效的：「动了很多次没结果」这类结构缺陷已经被挡在关卡集之外。<b>难度评分那一格是这两轮真正的发现</b>：机械上差着三倍的关卡，它一律给 ${R1.flat[0]} 分。更早的记录里这项的复测可信度是 0.93，那是个更小的样本——在这一批 ${pairs.length} 对复测上实测是 ${f2(retest(r => Number(r.m.rating?.difficulty)).r)}，所以它既不区分、也不像以为的那么稳。</p>
 
 <h2>两轮之间改了什么，效果如何</h2>
-<p>第一轮唯一可用的读数是「整批偏易」，加上抱怨里反复出现的「转了没反应」。据此只改了两件事：给<b>连续空转</b>设闸门（最多一次转动没反应，此前有 11 关能连着两次），以及把难度曲线<b>写成旋转深度的配额</b>而不是指望它从排序里自己长出来。</p>
+<p>第一轮唯一可用的读数是「整批偏易」，加上抱怨里反复出现的「转了没反应」。据此只改了两件事：给<b>连续空转</b>设闸门（最多一次转动没反应，此前有 11 关能连着两次），以及把难度曲线<b>写成旋转深度的配额</b>而不是指望它从排序里自己长出来。第三轮又只改了一件：桶内改按<b>反差</b>取。每轮一个变量，才看得出是哪个变量起的作用。</p>
 <table>
 <thead><tr><th>同一个规则机器人，两套关卡集</th><th class="n">第一版</th><th class="n">第二版</th></tr></thead>
 <tbody>
@@ -199,17 +221,21 @@ ${botOld && botNew ? `
 </table>
 <p>要做的事一样多，浪费掉的动作少了一半。这是同一个玩家的对照，所以变的是关卡不是模型的心情。</p>
 <table>
-<thead><tr><th>grok 实测，按档</th><th class="n">难度</th><th class="n">无解感</th><th class="n">盲转/关</th><th class="n">旋转深度</th></tr></thead>
+<thead><tr><th>grok 实测，按档（每关计一次，与它自己玩的那一版对齐）</th>${present.map(id => `<th class="n" colspan="2">${id}</th>`).join('')}</tr>
+<tr><th></th>${present.map(() => '<th class="n">难度</th><th class="n">无解感</th>').join('')}</tr></thead>
 <tbody>
-${BANDS.map(([label, a, b]) => {
-  const rows = bandOf(r2, a, b);
-  const g = k => mean(rows.map(r => Number(r.m.rating?.[k])).filter(Number.isFinite));
-  const rot = mean(levels.slice(a, b).map(l => l.campaign?.rotations ?? 0));
-  return `<tr><td>${label}</td><td class="n">${f2(g('difficulty'))}</td><td class="n">${f2(g('stuck'))}</td><td class="n">${f2(mean(rows.map(r => r.m.blindRotations ?? 0)))}</td><td class="n">${f1(rot)}</td></tr>`;
-}).join('\n')}
+${BANDS.map(([label, a, b]) => `<tr><td>${label}</td>${present.map(id => {
+  const st = bandStats(id, a, b);
+  return `<td class="n">${f2(st.difficulty)}</td><td class="n">${f2(st.stuck)}</td>`;
+}).join('')}</tr>`).join('\n')}
+<tr><td>后档的关卡参数</td>${present.map(id => {
+  const st = bandStats(id, 32, 50);
+  return `<td class="n">${f1(st.rotations)} 转</td><td class="n">反差 ${f1(st.contrast)}</td>`;
+}).join('')}</tr>
 </tbody>
 </table>
-<p>第一轮三档的难度评分是一条平线。第二轮后档升到了 ${f2(mean(bandOf(r2, 32, 50).map(r => Number(r.m.rating?.difficulty)).filter(Number.isFinite)))}，无解感也跟着抬起来——<b>这是评分第一次表现出梯度</b>，但只在按档平均之后。</p>
+<p>第一轮三档几乎是一条平线。第二轮把后档的旋转深度从 ${f1(bandStats(present[0], 32, 50).rotations)} 提到 ${f1(bandStats(present[1] ?? present[0], 32, 50).rotations)} 转，后档评分随之升到 ${f2(bandStats(present[1] ?? present[0], 32, 50).difficulty)}——<b>评分第一次表现出梯度</b>，但只在按档平均之后。</p>
+<p><b>第三轮是一次没被证实的假设。</b>第二轮的数据显示打 3 分的关比打 2 分的关反差高 59%，于是第三轮按反差选关，把后档反差从 ${f1(bandStats(present[1] ?? present[0], 32, 50).contrast)} 提到 ${f1(bandStats(LATEST, 32, 50).contrast)}、旋转深度不变。后档评分没有升，反而是 ${f2(bandStats(LATEST, 32, 50).difficulty)}。以这项评分 r = ${f2(retest(r => Number(r.m.rating?.difficulty)).r)} 的复测可信度，十几关上的这点差距读不出胜负——<b>该说的是「两版分不出来」，不是「反差有用」，也不是「反差没用」</b>。第三版留下来的理由是设计上的：反差是策划点名的好玩机制，而且它的家族分布更均衡（${present.length >= 2 ? '两家族 30/50 → 26/50' : ''}），这两条要靠人工审核裁定。</p>
 
 <h2>哪些信号复测得住</h2>
 <p>下面的 r 是在<b>这一批关卡</b>上现算的：同一个 agent 把每关玩了两次，${pairs.length} 对。不是引用更早、更小的样本。</p>

scripts/build-review-50.mjs

@@ -14,7 +14,7 @@ import { layout } from '../src/geometry.js';
 import { metrics } from '../src/bench.js';
 
 const args = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
-const AGENT = args.agent ?? 'R1 · grok xhigh';
+const AGENT = args.agent ?? 'R3 · grok xhigh';
 const OUT = resolve(args.out ?? 'docs/review-50.html');
 
 const raw = JSON.parse(readFileSync(resolve('levels/campaign-50.json'), 'utf8'));

scripts/round-set.mjs

@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+// Rebuild the level set a bench round actually played, from the round's runs.
+//
+//   node scripts/round-set.mjs --round=R2 [--out=…]
+//
+// Disk snapshots of "the set round N played" are not trustworthy: select-50
+// writes levels/campaign-50.json in place, so any selection experiment run
+// between the round and the copy silently replaces it. One such copy kept only
+// 28 of the 50 boards its round had played, which would have made the round
+// comparison compare a set against somebody else's levels.
+//
+// Runs store the whole board and its campaign index, so the round is its own
+// record. That record cannot drift.
+import { readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { deepLevelKey } from '../src/deep-lock.js';
+
+const args = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
+const ROUND = String(args.round ?? 'R2');
+
+export function roundSet(round, dir = resolve('bench/runs')) {
+  const byBoard = new Map();
+  for (const f of readdirSync(dir)) {
+    if (!f.endsWith('.json') || f === 'index.json') continue;
+    try {
+      const d = JSON.parse(readFileSync(resolve(dir, f), 'utf8'));
+      if (d.status === 'playing' || !d.level?.tiles) continue;
+      if (!new RegExp(`^${round} · `).test(String(d.agent))) continue;
+      const k = deepLevelKey(d.level);
+      if (!byBoard.has(k)) byBoard.set(k, { level: d.level, runs: [] });
+      byBoard.get(k).runs.push(d);
+    } catch { /* torn read */ }
+  }
+  return [...byBoard.values()].sort((a, b) => (a.level.campaign?.index ?? 0) - (b.level.campaign?.index ?? 0));
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const rows = roundSet(ROUND);
+  const levels = rows.map(r => r.level);
+  console.log(`${ROUND}：${levels.length} 关，共 ${rows.reduce((s, r) => s + r.runs.length, 0)} 局`);
+  console.log(`旋转深度 ${JSON.stringify(levels.reduce((m, l) => (m[l.campaign?.rotations ?? '?'] = (m[l.campaign?.rotations ?? '?'] || 0) + 1, m), {}))}`);
+  if (args.out) { writeFileSync(resolve(args.out), JSON.stringify({ generated: `round-set ${ROUND}`, count: levels.length, levels }, null, 1)); console.log(`已写入 ${args.out}`); }
+}