docs(levels): report what round 2 measured, including the number it overturns

The round accidentally ran every level twice, which turned into the most useful
thing it produced: 45 retest pairs on this exact set. 84% of them replayed the
identical action sequence, so the agent is near-deterministic and a second run
of a level buys almost nothing — the mechanical metrics retest at r = 1.00 for
that reason, not because they survived a hard test.

The ratings do not. Difficulty retests at r = 0.55 here, against the 0.93 this
project had on record from a smaller sample, and every pair that disagreed had
played the same trajectory: same moves, same outcome, different number. The
report now prints the reliability it measured rather than quoting the older
figure, and says which one it replaces.

Averaged by band over ~20 runs, though, the ratings finally track the curve —
2.00 / 2.05 / 2.40 with stuck at 1.21 / 1.15 / 1.74, where round 1 was a flat
line. So the rule is not "ratings are useless", it is "ratings are a band
statistic, never a per-level rank", and that is what the rules section says.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

scripts/build-process-report.mjs

@@ -7,7 +7,7 @@ import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { trace } from '../src/core.js';
 import { metrics } from '../src/bench.js';
-import { legibility } from '../src/deep-lock.js';
+import { deepLevelKey, legibility } from '../src/deep-lock.js';
 
 const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
 const mean = a => a.length ? a.reduce((x, y) => x + y, 0) / a.length : null;
@@ -29,11 +29,38 @@ for (const f of existsSync(resolve('bench/runs')) ? readdirSync(resolve('bench/r
     runs.push({ raw: d, m: metrics(d) });
   } catch { /* torn read */ }
 }
-const r1 = runs.filter(r => r.raw.agent === 'R1 · grok xhigh' && byId.has(r.raw.level?.id));
-const rated = r1.filter(r => r.m.rating && Number.isFinite(Number(r.m.rating.difficulty)));
+// Rounds are matched to their own level set by board, not by id: the reselect
+// between them reissued ids from the ordinal.
+const inSet = new Set(levels.map(deepLevelKey));
+const oldLevels = (() => { try { return JSON.parse(readFileSync(resolve('levels/campaign-50-r1.json'), 'utf8')).levels; } catch { return []; } })();
+const inOld = new Set(oldLevels.map(deepLevelKey));
+const round = (agent, set) => runs.filter(r => r.raw.agent === agent && r.raw.level?.tiles && set.has(deepLevelKey(r.raw.level)));
+const r1 = round('R1 · grok xhigh', inOld);
+const r2 = round('R2 · grok xhigh', inSet);
+const rated = r2.filter(r => r.m.rating && Number.isFinite(Number(r.m.rating.difficulty)));
 const byDiff = rated.reduce((m, r) => (m[r.m.rating.difficulty] = (m[r.m.rating.difficulty] || 0) + 1, m), {});
 const topDiff = Object.entries(byDiff).sort((a, b) => b[1] - a[1])[0] ?? ['—', 0];
 
+// Retest: the same agent played each level twice, so reliability is measurable
+// on this very set instead of quoted from an older, smaller one.
+const pairsBy = new Map();
+for (const r of r2) { const k = deepLevelKey(r.raw.level); if (!pairsBy.has(k)) pairsBy.set(k, []); pairsBy.get(k).push(r); }
+const pairs = [...pairsBy.values()].filter(v => v.length >= 2).map(v => [v[0], v[1]]);
+const sd = a => { const m = mean(a); return Math.sqrt(mean(a.map(v => (v - m) ** 2))); };
+const pearson = (A, B) => { const sa = sd(A), sb = sd(B); if (!sa || !sb) return null; const ma = mean(A), mb = mean(B); return mean(A.map((v, i) => (v - ma) * (B[i] - mb))) / (sa * sb); };
+const retest = get => {
+  const ok = pairs.filter(p => Number.isFinite(get(p[0])) && Number.isFinite(get(p[1])));
+  if (ok.length < 8) return { r: null, n: ok.length };
+  return { r: pearson(ok.map(p => get(p[0])), ok.map(p => get(p[1]))), n: ok.length };
+};
+const sig = r => (r.raw.steps ?? []).map(s => `${s.action?.type}:${s.action?.id ?? s.action?.direction ?? ''}`).join('|');
+const identical = pairs.filter(([a, b]) => sig(a) === sig(b)).length;
+const ratingDisagreed = pairs.filter(([a, b]) => a.m.rating?.difficulty != null && b.m.rating?.difficulty != null && a.m.rating.difficulty !== b.m.rating.difficulty);
+const disagreedOnSamePlay = ratingDisagreed.filter(([a, b]) => sig(a) === sig(b)).length;
+
+const bandOf = (rows, a, b) => rows.filter(r => { const i = levels.findIndex(l => deepLevelKey(l) === deepLevelKey(r.raw.level)); return i >= a && i < b; });
+const BANDS = [['前档 L1–L12', 0, 12], ['中档 L13–L32', 12, 32], ['后档 L33–L50', 32, 50]];
+
 const longest = l => Math.max(...l.tiles.map(t => trace(l, t, [], true).steps.length));
 const c50Longest = Math.max(...levels.map(longest));
 const loadPool = p => { try { const r = JSON.parse(readFileSync(resolve(p), 'utf8')); return Array.isArray(r) ? r : r.levels; } catch { return []; } };
@@ -43,25 +70,35 @@ const traps = levels.map(l => legibility(l).grazeTraps);
 
 const decisions = rated.map(r => r.m.decisions ?? 0);
 const R1 = {
-  n: r1.length, rated: rated.length,
-  cleared: r1.filter(r => r.m.cleared).length,
-  dead: mean(r1.map(r => r.m.deadTaps ?? 0)),
-  blind: mean(r1.map(r => r.m.blindRotations ?? 0)),
-  wrong: mean(r1.map(r => r.m.wrongWayRotations ?? 0)),
+  n: r2.length, rated: rated.length,
+  cleared: r2.filter(r => r.m.cleared).length,
+  dead: mean(r2.map(r => r.m.deadTaps ?? 0)),
+  blind: mean(r2.map(r => r.m.blindRotations ?? 0)),
+  wrong: mean(r2.map(r => r.m.wrongWayRotations ?? 0)),
   flat: topDiff, decMin: Math.min(...decisions), decMax: Math.max(...decisions),
+  r1n: r1.length, r1blind: mean(r1.map(r => r.m.blindRotations ?? 0)),
+  r1diff: mean(r1.map(r => Number(r.m.rating?.difficulty)).filter(Number.isFinite)),
 };
+// The rule bot over both sets: one player, two level sets, so the change is the
+// levels and not the mood of a model.
+const bot = tag => runs.filter(r => String(r.raw.run ?? '').startsWith(tag));
+const BOT = tag => { const v = bot(tag); return v.length ? {
+  n: v.length, blind: mean(v.map(r => r.m.blindRotations ?? 0)), dead: mean(v.map(r => r.m.deadTaps ?? 0)),
+  streak: Math.max(...v.map(r => r.m.longestBlindStreak ?? 0)), dec: mean(v.map(r => r.m.decisions ?? 0)),
+} : null; };
+const botOld = BOT('r1bot-'), botNew = BOT('r2bot-');
 
 // The rules, each with the observation that forced it. A rule without one is a
 // preference, and preferences do not belong in a process document.
 const RULES = [
-  { n: 1, rule: 'agent 轮次用来抓缺陷，不用来排难度',
-    ev: `R1 的 ${R1.rated} 关里有 ${R1.flat[1]} 关难度评分是 ${R1.flat[0]} 分，而这些关卡的决策数从 ${R1.decMin} 到 ${R1.decMax}。评分不是不准，是不区分。`,
-    why: '文字盘面把每颗棋子的路线、挡路者、紧贴者列成清单，agent 不必用眼睛顺线走一遍——而那正是这游戏的难点。agent 自己写过：「挡路名单已经写全，不用顺着格子数」。',
-    now: '难度排序改为机械假设（旋转深度、棋子数、互锁），由人工审核裁定；agent 轮次只看死局、盲转、转错、空点、演出。' },
-  { n: 2, rule: '先用规则机器人扫，再花 agent 时间',
-    ev: '机械旋转指标已被证明测的是关卡而不是玩家：4 次运行、3 家厂商，18 关里 17 关结论一致。',
-    why: '既然同一关在不同模型下得出同样的机械读数，那这个读数不需要模型来产生。',
-    now: '12 秒的规则机器人扫描替代一整轮 agent 跑批；agent 只用于抽样确认和读自由文本抱怨。' },
+  { n: 1, rule: '难度评分只能按档看，不能逐关看',
+    ev: `${R1.rated} 局里有 ${R1.flat[1]} 局打的是 ${R1.flat[0]} 分，而这些关卡的决策数从 ${R1.decMin} 到 ${R1.decMax}。逐关复测 r = ${f2(retest(r => Number(r.m.rating?.difficulty)).r)}，而且 ${ratingDisagreed.length} 对分歧里有 ${disagreedOnSamePlay} 对<b>走的是同一条轨迹</b>——同样的动作、同样的结果，模型给自己打了不同的分。但按档平均约 20 局后，曲线第一次显形了（见上表）。`,
+    why: '文字盘面把每颗棋子的路线、挡路者、紧贴者列成清单，agent 不必用眼睛顺线走一遍——而那正是这游戏的难点。agent 自己写过：「挡路名单已经写全，不用顺着格子数」。剩下的评分噪声，只有靠聚合才压得下去。',
+    now: '逐关排序改为机械假设（旋转深度、棋子数、互锁），由人工审核裁定；评分只用来读整档的高低。' },
+  { n: 2, rule: '先用规则机器人扫，再花 agent 时间；同一关不要跑两遍',
+    ev: `第二轮意外把每关跑了两次，得到 ${pairs.length} 对复测：其中 ${identical} 对（${pct(identical, pairs.length)}）重放了<b>完全相同</b>的动作序列，机械指标复测 r = 1.00。规则机器人跑完 50 关只要 1 秒。`,
+    why: 'agent 近乎确定性，重复同一关几乎不产生新信息。为「提高可信度」而重复跑，是把预算花在已经知道的答案上。',
+    now: '全量验收交给规则机器人；agent 每关只跑一次，省下的预算用来多覆盖关卡。' },
   { n: 3, rule: '门槛写在生成期，不写在筛选期',
     ev: '把开局门槛提到「可点≥3 枚、演出≥0.8s」后筛选写出了 0 关——池子的中位数是 2 枚 / 0.57s。',
     why: '筛选只能从池子里挑，挑不出池子里没有的东西。分布不对时，要改的是构造。',
@@ -135,9 +172,9 @@ a{color:var(--rail)}
 <div class="wrap">
 <h1>关卡制作流程复盘</h1>
 <p class="lede">这一轮做 50 关的过程里，最有用的结论是一个<b>否定结论</b>：原计划的「agent 打分 → 拟合权重 → 重选关卡」这个闭环<b>合不上</b>，而且不是样本量的问题。下面是它为什么合不上，以及流程该改成什么样。</p>
-<p class="meta">实测批次 R1 · grok-4.6 xhigh · ${R1.n} 关 · 生成于 ${new Date().toLocaleString('sv-SE').slice(0, 16)}</p>
+<p class="meta">实测批次 R2 · grok-4.6 xhigh · ${R1.n} 局（50 关，多数跑了两次）· 对照批次 R1 ${R1.r1n} 局 · 生成于 ${new Date().toLocaleString('sv-SE').slice(0, 16)}</p>
 
-<h2>R1 实测到了什么</h2>
+<h2>第二轮实测到了什么</h2>
 <div class="tiles">
   <div class="tile ${R1.cleared === R1.n ? 'ok' : 'miss'}"><div class="k">通关</div><div class="v">${R1.cleared}/${R1.n}</div><div class="n">无死局</div></div>
   <div class="tile ${R1.dead === 0 ? 'ok' : 'miss'}"><div class="k">空点 / 关</div><div class="v">${f2(R1.dead)}</div><div class="n">目标 0</div></div>
@@ -145,7 +182,56 @@ a{color:var(--rail)}
   <div class="tile ${R1.wrong < .5 ? 'ok' : 'miss'}"><div class="k">转错 / 关</div><div class="v">${f2(R1.wrong)}</div><div class="n">目标 0</div></div>
   <div class="tile miss"><div class="k">难度评分</div><div class="v">${R1.flat[1]}/${R1.rated}</div><div class="n">都是 ${R1.flat[0]} 分 — 不区分</div></div>
 </div>
-<p>机械面干净，说明生成期的闸门是有效的：「动了很多次没结果」这类结构缺陷已经被挡在关卡集之外。<b>难度评分那一格是这一轮真正的发现</b>——它不是不准（复测 r = 0.93，相当稳），它是<b>不区分</b>：机械上差着三倍的关卡，它一律给 ${R1.flat[0]} 分。</p>
+<p>机械面干净，说明生成期的闸门是有效的：「动了很多次没结果」这类结构缺陷已经被挡在关卡集之外。<b>难度评分那一格是这两轮真正的发现</b>：机械上差着三倍的关卡，它一律给 ${R1.flat[0]} 分。更早的记录里这项的复测可信度是 0.93，那是个更小的样本——在这一批 ${pairs.length} 对复测上实测是 ${f2(retest(r => Number(r.m.rating?.difficulty)).r)}，所以它既不区分、也不像以为的那么稳。</p>
+
+<h2>两轮之间改了什么，效果如何</h2>
+<p>第一轮唯一可用的读数是「整批偏易」，加上抱怨里反复出现的「转了没反应」。据此只改了两件事：给<b>连续空转</b>设闸门（最多一次转动没反应，此前有 11 关能连着两次），以及把难度曲线<b>写成旋转深度的配额</b>而不是指望它从排序里自己长出来。</p>
+<table>
+<thead><tr><th>同一个规则机器人，两套关卡集</th><th class="n">第一版</th><th class="n">第二版</th></tr></thead>
+<tbody>
+${botOld && botNew ? `
+<tr><td>盲转 / 关</td><td class="n">${f2(botOld.blind)}</td><td class="n yes">${f2(botNew.blind)}</td></tr>
+<tr><td>最长连续盲转</td><td class="n">${botOld.streak}</td><td class="n yes">${botNew.streak}</td></tr>
+<tr><td>空点 / 关</td><td class="n">${f2(botOld.dead)}</td><td class="n">${f2(botNew.dead)}</td></tr>
+<tr><td>决策 / 关（要做的事）</td><td class="n">${f1(botOld.dec)}</td><td class="n">${f1(botNew.dec)}</td></tr>
+` : '<tr><td colspan="3">规则机器人数据缺失</td></tr>'}
+</tbody>
+</table>
+<p>要做的事一样多，浪费掉的动作少了一半。这是同一个玩家的对照，所以变的是关卡不是模型的心情。</p>
+<table>
+<thead><tr><th>grok 实测，按档</th><th class="n">难度</th><th class="n">无解感</th><th class="n">盲转/关</th><th class="n">旋转深度</th></tr></thead>
+<tbody>
+${BANDS.map(([label, a, b]) => {
+  const rows = bandOf(r2, a, b);
+  const g = k => mean(rows.map(r => Number(r.m.rating?.[k])).filter(Number.isFinite));
+  const rot = mean(levels.slice(a, b).map(l => l.campaign?.rotations ?? 0));
+  return `<tr><td>${label}</td><td class="n">${f2(g('difficulty'))}</td><td class="n">${f2(g('stuck'))}</td><td class="n">${f2(mean(rows.map(r => r.m.blindRotations ?? 0)))}</td><td class="n">${f1(rot)}</td></tr>`;
+}).join('\n')}
+</tbody>
+</table>
+<p>第一轮三档的难度评分是一条平线。第二轮后档升到了 ${f2(mean(bandOf(r2, 32, 50).map(r => Number(r.m.rating?.difficulty)).filter(Number.isFinite)))}，无解感也跟着抬起来——<b>这是评分第一次表现出梯度</b>，但只在按档平均之后。</p>
+
+<h2>哪些信号复测得住</h2>
+<p>下面的 r 是在<b>这一批关卡</b>上现算的：同一个 agent 把每关玩了两次，${pairs.length} 对。不是引用更早、更小的样本。</p>
+<table>
+<thead><tr><th>信号</th><th class="n">复测 r</th><th class="n">n</th><th>怎么用</th></tr></thead>
+<tbody>
+${[['盲转次数', r => r.m.blindRotations, '可用：驱动了真实修复'],
+   ['转错方向次数', r => r.m.wrongWayRotations, '可用'],
+   ['实际旋转数', r => r.m.rotations, '可用'],
+   ['步数', r => r.m.steps, '可用'],
+   ['决策数', r => r.m.decisions, '可用'],
+   ['无解感评分', r => Number(r.m.rating?.stuck), '只按档看'],
+   ['难度评分', r => Number(r.m.rating?.difficulty), '只按档看，约 20 局起'],
+   ['好玩评分', r => Number(r.m.rating?.fun), '不可用'],
+   ['清晰度评分', r => Number(r.m.rating?.clarity), '不可用']].map(([name, get, use]) => {
+  const t = retest(get);
+  const cls = t.r == null ? '' : t.r >= 0.9 ? 'yes' : t.r >= 0.45 ? '' : 'no';
+  return `<tr><td>${name}</td><td class="n ${cls}">${t.r == null ? '—' : f2(t.r)}</td><td class="n">${t.n}</td><td>${use}</td></tr>`;
+}).join('\n')}
+</tbody>
+</table>
+<p>机械指标全部 r ≈ 1.00，但那多半是因为 agent <b>${pct(identical, pairs.length)} 的情况下重放了完全相同的动作序列</b>——这本身才是有用的那条结论：重复跑同一关买不到新信息。评分则相反：同一条轨迹，${disagreedOnSamePlay} 对给出了不同的难度分。</p>
 
 <h2>五条规则，各自的证据</h2>
 ${RULES.map(r => `<div class="rule">
@@ -161,12 +247,12 @@ ${RULES.map(r => `<div class="rule">
 <table>
 <thead><tr><th>指标</th><th class="n">可用</th><th>依据</th></tr></thead>
 <tbody>
-<tr><td>盲转 / 多余转 / 转错方向</td><td class="n yes">可用</td><td>复测 r ≈ 0.82–0.92；驱动了真实修复（1.17 / 1.22 / 0.44 → 0）</td></tr>
+<tr><td>盲转 / 多余转 / 转错方向</td><td class="n yes">可用</td><td>本批复测 r = ${f2(retest(r => r.m.blindRotations).r)}；驱动了真实修复（盲转 1.17 → ${f2(R1.blind)}）</td></tr>
 <tr><td>是否通关、空点次数</td><td class="n yes">可用</td><td>直接观察，无主观成分</td></tr>
 <tr><td>自由文本抱怨</td><td class="n yes">可用</td><td>33 条里两个主题高度重复，指标抓不到；但须对照基线（规则 5）</td></tr>
-<tr><td>难度评分</td><td class="n no">只能看总体</td><td>稳定但不区分；单关排序不可用，整批读数（偏易一档）可用</td></tr>
-<tr><td>无解感评分</td><td class="n">谨慎</td><td>复测 r = 0.85，与无效动作相关 0.66，可当挫败感的代理</td></tr>
-<tr><td>好玩 / 清晰度 / 旋转评分</td><td class="n no">不可用</td><td>复测 r ≤ 0.56，是模型噪声</td></tr>
+<tr><td>难度评分</td><td class="n no">只按档看</td><td>逐关复测 r = ${f2(retest(r => Number(r.m.rating?.difficulty)).r)}，且同一条轨迹会打出不同分；按档平均约 20 局才显出梯度</td></tr>
+<tr><td>无解感评分</td><td class="n">只按档看</td><td>复测 r = ${f2(retest(r => Number(r.m.rating?.stuck)).r)}；按档看时与难度同向抬升，可当挫败感的代理</td></tr>
+<tr><td>好玩 / 清晰度评分</td><td class="n no">不可用</td><td>复测 r = ${f2(retest(r => Number(r.m.rating?.fun)).r)} / ${f2(retest(r => Number(r.m.rating?.clarity)).r)}，是模型噪声</td></tr>
 </tbody>
 </table>