feat(bench): fit difficulty from a played round, and report when it cannot

refit-weights regresses the weights select-50 scores with against what agents actually experienced, so a round of play is a flag on the next selection rather than an edit. Ridge, because seven collinear features on fifty levels otherwise hand back large coefficients that cancel. Most of the work here is in refusing to produce a number. The response averages the difficulty rating with two continuous measures of the same thing, and a component with no spread is dropped by name instead of being silently averaged in as zeros — which is what the first version did, reporting R² = 0.75 for a fit that never saw a rating. When the rating is flat the script now says so and says what the remaining fit is really measuring, which is board size. legibility() asks what it costs to know what you may tap: how far the eye must follow the cheapest free piece, and how many pieces are held by somebody their line never touches. It was built to explain why ratings came back flat. It does not — it correlates -0.02 with them — so it stays a measurement, printed on the review sheet beside the longest route on the board, and gates nothing. prototype-d5 takes --seedFrom so a wide sweep shards across cores. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-14 17:03:43 +08:00
parent b35ea201db
commit d374c208f0
10 changed files with 3886 additions and 287 deletions
@@ -9,6 +9,7 @@
 import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { trace, rotateBoard } from '../src/core.js';
+import { legibility } from '../src/deep-lock.js';
 import { layout } from '../src/geometry.js';
 import { metrics } from '../src/bench.js';

@@ -62,6 +63,15 @@ function thumbnail(level, size = 168) {
    + `<g class="cells">${cells}</g><g class="pieces">${pieces}</g></svg>`;
 }

+// The two things the agent complained about in all 33 reports it filed, and the
+// two the text bench is least able to judge for us: how long the longest line
+// on the board is, and how many pieces are held by somebody their line never
+// touches. Both are perceptual, so they are printed rather than gated on.
+const perceptual = Object.fromEntries(levels.map(l => [l.id, {
+  longest: Math.max(...l.tiles.map(t => trace(l, t, [], true).steps.length)),
+  ...legibility(l),
+}]));
+
 const rated = levels.map(l => verdicts[l.id]).filter(Boolean);
 const band = (a, b) => levels.slice(a, b).map(l => verdicts[l.id]).filter(Boolean);
 const bandRow = (label, a, b) => {
@@ -116,6 +126,7 @@ td.n,th.n{text-align:right;font-family:ui-monospace,monospace;white-space:nowrap
 .thumb text{fill:#fff;text-anchor:middle;font-family:system-ui}
 .kv{display:grid;grid-template-columns:auto auto;gap:1px 8px;font-size:11.5px;color:var(--ink2)}
 .kv b{font-family:ui-monospace,monospace;color:var(--ink);font-weight:600;text-align:right}
+.kv b.hot{color:var(--warn)}
 .verdict{font-size:11.5px;border-top:1px dashed var(--rule);padding-top:7px;color:var(--ink2)}
 .verdict .row{display:flex;gap:9px;flex-wrap:wrap;font-family:ui-monospace,monospace}
 .verdict .worst{margin-top:4px;color:var(--ink3);line-height:1.5}
@@ -123,6 +134,9 @@ td.n,th.n{text-align:right;font-family:ui-monospace,monospace;white-space:nowrap
 .pill{display:inline-block;font-size:10.5px;padding:1px 7px;border-radius:20px;font-weight:600}
 .pill.bad{background:var(--bad);color:#fff}
 .pill.warn{background:var(--warn);color:#fff}
+.note{background:var(--panel);border:1px solid var(--rule);border-left:3px solid var(--rail);border-radius:10px;padding:4px 18px;margin:12px 0 8px}
+.note p{font-size:13.5px;color:var(--ink2);max-width:78ch}
+.note b{color:var(--ink)}
 footer{margin-top:50px;padding-top:18px;border-top:1px solid var(--rule);color:var(--ink3);font-size:12px}
 </style>
 </head>
@@ -143,6 +157,14 @@ ${bandRow('后档 L33–L50', 32, 50)}
 </table>
 <p class="meta">难度 3 = 刚刚好；无解感越低越好。这两项是评分里唯二复测可信的（r = 0.93 / 0.85），其余几项是模型噪声，不列。</p>

+<h2>这批数字里，哪些能信</h2>
+<div class="note">
+<p><b>机械面是干净的，可以信。</b>agent 跑完 ${rated.length} 关：空点 ${f1(mean(rated.map(m => m.deadTaps)))} 次/关、盲转 ${f1(mean(rated.map(m => m.blindRotations)))} 次/关、反向转 ${f1(mean(rated.map(m => m.wrongWayRotations)))} 次/关，${rated.filter(m => m.cleared).length}/${rated.length} 关通关。「动了半天没结果」这类结构缺陷已经被生成期的闸门挡掉了。</p>
+<p><b>难度评分不能信，它区分不了这批关卡。</b>${rated.length} 关里有 ${rated.filter(m => Number(m.rating?.difficulty) === 2).length} 关被打成 2 分，而这些关卡机械上差着三倍（决策数 0–8）。原因不在关卡，在 bench：文字盘面会把每颗棋子的路线、挡路者、紧贴者<b>直接列成清单</b>，agent 从来不需要用眼睛顺着线走一遍——而那正是这个游戏的难点所在。agent 自己在报告里写过这句：「挡路名单已经写全，不用顺着格子数」。</p>
+<p><b>所以难度顺序是机械假设，等你的判定。</b>L1 到 L50 是按旋转深度、棋子数、互锁程度排的，不是按谁测出来更难。评分唯一能用的是它的<b>总体读数</b>：整批偏易一档（2 分 vs 刚刚好的 3 分）。</p>
+<p><b>两件事我没有擅自改，留给你定。</b>agent 的 33 条抱怨里反复出现两个主题：路线太长看不完（「68 格 27 弯」「42 格 42 弯」），以及紧贴容易看漏（「看着能出界却会擦到」）。两项都是感知问题，而<b>已上线的 v0.5 在这两项上更严重</b>（最长路线 88 格 vs 这里 ${Math.max(...Object.values(perceptual).map(p => p.longest))} 格）——它是这个游戏本来的性格，不是这次构造引入的。卡片上「最长路线」超 45 格、「紧贴棋子」超 1 枚会标黄，你试玩时留意。</p>
+</div>
+
 <h2>逐关</h2>
 <div class="grid">
 ${levels.map((l, i) => {
@@ -160,6 +182,8 @@ ${levels.map((l, i) => {
    <span>开局可点</span><b>${c.openingChoices ?? '?'} 枚</b>
    <span>开局演出</span><b>${c.openingBestMs != null ? (c.openingBestMs / 1000).toFixed(1) + 's' : '?'}</b>
    <span>长巡游 / 反差</span><b>${c.cruises ?? '?'} · ${c.contrast ?? '?'}</b>
+    <span>最长路线</span><b class="${perceptual[l.id].longest > 45 ? 'hot' : ''}">${perceptual[l.id].longest} 格</b>
+    <span>紧贴棋子</span><b class="${perceptual[l.id].grazeTraps > 1 ? 'hot' : ''}">${perceptual[l.id].grazeTraps} 枚</b>
  </div>
  ${v ? `<div class="verdict">
    <div class="row"><span>难度 ${v.rating?.difficulty ?? '—'}</span><span>无解感 ${v.rating?.stuck ?? '—'}</span><span>${v.steps} 步</span><span>${v.decisions} 决策</span></div>
@@ -10,6 +10,8 @@ import { createDeepLock, growDeepLevel, deepLevelStats, deepLevelKey, deadTurnAu

 const args = Object.fromEntries(process.argv.slice(2).map(a => { const [k, v] = a.replace(/^--/, '').split('='); return [k, v ?? true]; }));
 const SEEDS = Number(args.seeds ?? 8), TARGET = Number(args.target ?? 12), KEEP = Number(args.keep ?? 18);
+// Seeds are independent, so a wide sweep shards across cores by seed range.
+const SEED_FROM = Number(args.seedFrom ?? 0);
 const OUT = resolve(args.out ?? 'levels/prototypes/d5.json');
 // 91-cell boards stay out by default: they are a readability problem, not a difficulty band.
 const RADII = String(args.radii ?? '3,4').split(',').map(Number);
@@ -41,7 +43,7 @@ const yieldTable = {};
 for (const { id: family } of ROUTE_FAMILIES) for (const radius of RADII) {
  const cellKey = `${family}/r${radius}`;
  yieldTable[cellKey] = { locks: 0, grown: 0, seeds: SEEDS };
-  for (let seed = 0; seed < SEEDS; seed++) {
+  for (let seed = SEED_FROM; seed < SEED_FROM + SEEDS; seed++) {
    const base = createRouteBoard(seed, { family, radius });
    const lock = createDeepLock(base, { seed, maxChecks: 600000 });
    if (!lock) continue;
@@ -0,0 +1,155 @@
+#!/usr/bin/env node
+// Refit the difficulty weights that select-50 scores with, from a round of
+// agent play.
+//
+//   node scripts/refit-weights.mjs [--agent="R1 · grok xhigh"] [--ridge=0.3]
+//
+// select-50 scores a candidate from mechanical features, but the weights on
+// those features started as guesses. This fits them against what agents
+// actually experienced, and prints the --weights flag for the next round.
+//
+// The response is not the difficulty rating alone. That rating is reliable
+// (r = 0.93 on retest) but it is a 1-5 integer, so on fifty levels it is nearly
+// constant and a fit against it alone would be fitting rounding. It is averaged
+// with two continuous measures of the same thing — how many decisions the level
+// forced, and how much action went nowhere — each standardised first.
+//
+// Ridge, because seven features on fifty levels with collinear inputs (bigger
+// boards hold more pieces) will otherwise hand back large coefficients that
+// cancel. The printed R² says whether the fit earned its keep; when it is near
+// zero, the honest move is to keep the old weights and say the round carried no
+// signal.
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { metrics } from '../src/bench.js';
+import { legibility } from '../src/deep-lock.js';
+
+const args = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
+const AGENT = args.agent ?? 'R1 · grok xhigh';
+const RIDGE = Number(args.ridge ?? 0.3);
+
+// The features select-50 scores with, plus the two the design corrections
+// argued matter — how much the opening offers, and how much a winding route
+// pays off over a straight one.
+const FEATURES = ['rotations', 'pieces', 'lockedShare', 'monkeyAPT', 'cells', 'contrast', 'openingChoices',
+  'glanceCost', 'medianVerify', 'grazeTraps'];
+const featureOf = level => {
+  const c = level.campaign ?? {};
+  const p = level.prototype ?? {};
+  return {
+    rotations: c.rotations ?? 3, pieces: c.pieces ?? level.tiles.length,
+    lockedShare: c.lockedShare ?? 0, monkeyAPT: p.monkeyActionsPerPiece ?? 0,
+    cells: level.cells.length, contrast: c.contrast ?? 0,
+    openingChoices: c.openingChoices ?? 0,
+    ...(({ glanceCost, medianVerify, grazeTraps }) => ({ glanceCost: glanceCost ?? 0, medianVerify, grazeTraps }))(legibility(level)),
+  };
+};
+
+const levels = JSON.parse(readFileSync(resolve('levels/campaign-50.json'), 'utf8')).levels;
+const byId = new Map(levels.map(l => [l.id, l]));
+
+const played = new Map();
+const RUNS = resolve('bench/runs');
+for (const f of existsSync(RUNS) ? readdirSync(RUNS) : []) {
+  if (!f.endsWith('.json') || f === 'index.json') continue;
+  try {
+    const d = JSON.parse(readFileSync(resolve(RUNS, f), 'utf8'));
+    if (d.agent !== AGENT || !byId.has(d.level?.id)) continue;
+    played.set(d.level.id, metrics(d));
+  } catch { /* torn read */ }
+}
+
+const rows = [...played].map(([id, m]) => ({ id, m, x: featureOf(byId.get(id)) }))
+  .filter(r => Number.isFinite(Number(r.m.rating?.difficulty)));
+if (rows.length < 12) {
+  console.error(`只有 ${rows.length} 关有评分，样本太少，不拟合。`);
+  process.exit(1);
+}
+
+const mean = a => a.reduce((x, y) => x + y, 0) / a.length;
+const sd = a => { const m = mean(a); return Math.sqrt(mean(a.map(v => (v - m) ** 2))); };
+const z = a => { const m = mean(a), s = sd(a); return s === 0 ? a.map(() => 0) : a.map(v => (v - m) / s); };
+
+// Three views of the same quantity: what the agent said, how much it had to
+// decide, and how much of what it did led nowhere.
+const parts = {
+  difficulty: rows.map(r => Number(r.m.rating.difficulty)),
+  decisions: rows.map(r => Math.log1p(r.m.decisions ?? 0)),
+  wasted: rows.map(r => Math.log1p((r.m.deadTaps ?? 0) + (r.m.blindRotations ?? 0) + (r.m.wrongWayRotations ?? 0))),
+};
+// A part with no spread contributes nothing but still divides the average, so
+// it is dropped by name rather than silently zeroed. This matters: a round
+// where every level came back rated 2 leaves an effort-only response, and
+// effort alone largely re-measures board size.
+const liveParts = Object.entries(parts).filter(([, v]) => sd(v) > 1e-9);
+const flatParts = Object.keys(parts).filter(k => !liveParts.some(([n]) => n === k));
+if (!liveParts.length) { console.error('本轮没有任何一项有方差，无从拟合。'); process.exit(1); }
+const zparts = liveParts.map(([, v]) => z(v));
+const y = rows.map((_, i) => mean(zparts.map(p => p[i])));
+const zy = z(y);
+
+// Drop features the round could not separate: a column that never varies
+// carries no information and a ridge fit would still hand it a coefficient.
+const live = FEATURES.filter(f => sd(rows.map(r => r.x[f])) > 1e-9);
+const dropped = FEATURES.filter(f => !live.includes(f));
+const cols = live.map(f => rows.map(r => r.x[f]));
+const sds = Object.fromEntries(live.map((f, j) => [f, sd(cols[j])]));
+const Z = live.map((_, j) => z(cols[j]));
+
+// beta = (Z'Z + λI)^-1 Z'y, solved by Gaussian elimination with partial pivoting.
+const p = live.length, n = rows.length;
+const A = Array.from({ length: p }, (_, i) => Array.from({ length: p + 1 }, (_, j) => {
+  if (j === p) return Z[i].reduce((s, v, k) => s + v * zy[k], 0) / n;
+  return Z[i].reduce((s, v, k) => s + v * Z[j][k], 0) / n + (i === j ? RIDGE : 0);
+}));
+for (let i = 0; i < p; i++) {
+  let piv = i;
+  for (let k = i + 1; k < p; k++) if (Math.abs(A[k][i]) > Math.abs(A[piv][i])) piv = k;
+  [A[i], A[piv]] = [A[piv], A[i]];
+  for (let k = 0; k < p; k++) {
+    if (k === i || A[i][i] === 0) continue;
+    const f = A[k][i] / A[i][i];
+    for (let j = i; j <= p; j++) A[k][j] -= f * A[i][j];
+  }
+}
+const beta = A.map((row, i) => row[i] === 0 ? 0 : row[p] / row[i]);
+
+const fitted = rows.map((_, k) => live.reduce((s, _f, j) => s + beta[j] * Z[j][k], 0));
+const ssRes = zy.reduce((s, v, k) => s + (v - fitted[k]) ** 2, 0);
+const r2 = 1 - ssRes / zy.reduce((s, v) => s + v * v, 0);
+
+// Back to raw-feature units, then rescaled so scores land in the same range as
+// the round just played — the score is only ever used to rank and to spread
+// picks evenly, so its scale is free, and holding it steady keeps the printed
+// band numbers comparable from round to round.
+const OLD = { rotations: 1.0, pieces: 0.10, lockedShare: 1.6, monkeyAPT: 0.14, cells: 0.004, contrast: 0, openingChoices: 0, glanceCost: 0, medianVerify: 0, grazeTraps: 0 };
+const raw = Object.fromEntries(live.map((f, j) => [f, beta[j] / sds[f]]));
+const scoreWith = w => rows.map(r => live.reduce((s, f) => s + (w[f] ?? 0) * r.x[f], 0));
+const gain = sd(scoreWith(OLD)) / (sd(scoreWith(raw)) || 1);
+const fit = Object.fromEntries(live.map(f => [f, Number((raw[f] * gain).toFixed(4))]));
+for (const f of dropped) fit[f] = OLD[f] ?? 0;
+
+const corr = (a, b) => { const za = z(a), zb = z(b); return mean(za.map((v, i) => v * zb[i])); };
+console.log(`样本 ${n} 关 · agent ${AGENT} · ridge ${RIDGE}`);
+const NAME = { difficulty: '评分难度', decisions: 'log 决策数', wasted: 'log 无效动作' };
+console.log(`合成难度 = ${liveParts.map(([k]) => 'z(' + NAME[k] + ')').join(' + ')} 的均值`);
+if (flatParts.length) {
+  console.log(`  !! ${flatParts.map(k => NAME[k]).join('、')} 本轮方差为 0，没参与合成`);
+  if (flatParts.includes('difficulty')) {
+    console.log('  !! 难度评分全同分 —— 这一轮拿不到难度信号，剩下的只是「这关有多少事要做」，');
+    console.log('     而那主要由棋子数和格子数决定。照这组权重重选，等于按盘面大小排序。');
+  }
+}
+for (const [k, v] of liveParts) for (const [k2, v2] of liveParts) if (k < k2) console.log(`  ${NAME[k]} × ${NAME[k2]}：${corr(v, v2).toFixed(2)}`);
+console.log(`拟合 R² = ${r2.toFixed(3)}${r2 < 0.25 ? '  ← 太低，这一轮没给出可用信号，别换权重' : ''}`);
+if (flatParts.includes('difficulty')) console.log('R² 高不代表可用：被拟合的是工作量，不是难度。');
+console.log('');
+console.log('| 特征 | 单独相关 | 标准化系数 | 旧权重 | 新权重 |');
+console.log('|---|---:|---:|---:|---:|');
+for (const f of FEATURES) {
+  const j = live.indexOf(f);
+  const c = j < 0 ? null : corr(rows.map(r => r.x[f]), y);
+  console.log(`| ${f} | ${c == null ? '—（本轮无变化）' : c.toFixed(2)} | ${j < 0 ? '—' : beta[j].toFixed(3)} | ${(OLD[f] ?? 0)} | ${fit[f]} |`);
+}
+console.log('');
+console.log(`--weights='${JSON.stringify(fit)}'`);