#!/usr/bin/env node // Build docs/level-process.html — what this round changed about how levels // get made. Every figure is measured here, not typed in. // // node scripts/build-process-report.mjs import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { trace } from '../src/core.js'; import { metrics } from '../src/bench.js'; import { roundSet } from './round-set.mjs'; import { deepLevelKey, legibility } from '../src/deep-lock.js'; const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c])); const mean = a => a.length ? a.reduce((x, y) => x + y, 0) / a.length : null; const f1 = x => x == null || Number.isNaN(x) ? '—' : (Math.round(x * 10) / 10).toFixed(1); const f2 = x => x == null || Number.isNaN(x) ? '—' : (Math.round(x * 100) / 100).toFixed(2); const pct = (a, b) => b ? Math.round(100 * a / b) + '%' : '—'; const levels = JSON.parse(readFileSync(resolve('levels/campaign-50.json'), 'utf8')).levels; const byId = new Map(levels.map(l => [l.id, l])); // Every round on record, so the claim "agent rounds catch defects" is shown // against the rounds rather than asserted. const runs = []; for (const f of existsSync(resolve('bench/runs')) ? readdirSync(resolve('bench/runs')) : []) { if (!f.endsWith('.json') || f === 'index.json') continue; try { const d = JSON.parse(readFileSync(resolve('bench/runs', f), 'utf8')); if (d.status === 'playing') continue; runs.push({ raw: d, m: metrics(d) }); } catch { /* torn read */ } } // Each round is rebuilt from its own runs. A disk snapshot of "the set round N // played" cannot be trusted — select-50 writes campaign-50.json in place, so a // selection experiment between the round and the copy replaces it silently, and // one such copy kept only 28 of its round's 50 boards. const ROUND_IDS = ['R1', 'R2', 'R3', 'R4']; const sets = Object.fromEntries(ROUND_IDS.map(id => [id, roundSet(id)]).filter(([, rows]) => rows.length)); const present = Object.keys(sets); const LATEST = present[present.length - 1] ?? 'R1'; const PREV = present[present.length - 2] ?? null; const flat = id => (sets[id] ?? []).flatMap(e => e.runs.map(raw => ({ raw, m: metrics(raw) }))); const r1 = PREV ? flat(PREV) : []; const r2 = flat(LATEST); const rated = r2.filter(r => r.m.rating && Number.isFinite(Number(r.m.rating.difficulty))); const byDiff = rated.reduce((m, r) => (m[r.m.rating.difficulty] = (m[r.m.rating.difficulty] || 0) + 1, m), {}); const topDiff = Object.entries(byDiff).sort((a, b) => b[1] - a[1])[0] ?? ['—', 0]; // Retest pairs, wherever a round happened to play a level twice. Round 2 did it // across the board by accident, which is where the reliability figures come from. const pairsBy = new Map(); for (const id of present) for (const e of sets[id]) { const rows = e.runs.map(raw => ({ raw, m: metrics(raw) })); if (rows.length >= 2) pairsBy.set(`${id}:${deepLevelKey(e.level)}`, rows); } const pairs = [...pairsBy.values()].map(v => [v[0], v[1]]); const sd = a => { const m = mean(a); return Math.sqrt(mean(a.map(v => (v - m) ** 2))); }; const pearson = (A, B) => { const sa = sd(A), sb = sd(B); if (!sa || !sb) return null; const ma = mean(A), mb = mean(B); return mean(A.map((v, i) => (v - ma) * (B[i] - mb))) / (sa * sb); }; const retest = get => { const ok = pairs.filter(p => Number.isFinite(get(p[0])) && Number.isFinite(get(p[1]))); if (ok.length < 8) return { r: null, n: ok.length }; return { r: pearson(ok.map(p => get(p[0])), ok.map(p => get(p[1]))), n: ok.length }; }; const sig = r => (r.raw.steps ?? []).map(s => `${s.action?.type}:${s.action?.id ?? s.action?.direction ?? ''}`).join('|'); const identical = pairs.filter(([a, b]) => sig(a) === sig(b)).length; const ratingDisagreed = pairs.filter(([a, b]) => a.m.rating?.difficulty != null && b.m.rating?.difficulty != null && a.m.rating.difficulty !== b.m.rating.difficulty); const disagreedOnSamePlay = ratingDisagreed.filter(([a, b]) => sig(a) === sig(b)).length; // A band is a slice of the round's own ordering, and each level counts once no // matter how many times it was played. const bandStats = (id, a, b) => { const slice = (sets[id] ?? []).slice(a, b); const per = get => slice.map(e => mean(e.runs.map(raw => get(metrics(raw))).filter(Number.isFinite))).filter(Number.isFinite); return { n: slice.length, difficulty: mean(per(m => Number(m.rating?.difficulty))), stuck: mean(per(m => Number(m.rating?.stuck))), blind: mean(per(m => m.blindRotations ?? 0)), contrast: mean(slice.map(e => e.level.campaign?.contrast ?? 0)), rotations: mean(slice.map(e => e.level.campaign?.rotations ?? 0)), }; }; const bandOf = (rows, a, b) => rows.filter(r => { const i = levels.findIndex(l => deepLevelKey(l) === deepLevelKey(r.raw.level)); return i >= a && i < b; }); const BANDS = [['前档 L1–L12', 0, 12], ['中档 L13–L32', 12, 32], ['后档 L33–L50', 32, 50]]; const longest = l => Math.max(...l.tiles.map(t => trace(l, t, [], true).steps.length)); const c50Longest = Math.max(...levels.map(longest)); const loadPool = p => { try { const r = JSON.parse(readFileSync(resolve(p), 'utf8')); return Array.isArray(r) ? r : r.levels; } catch { return []; } }; const v05 = loadPool('bench/pool-easy.json'); const v05Longest = v05.length ? Math.max(...v05.map(longest)) : null; const traps = levels.map(l => legibility(l).grazeTraps); const decisions = rated.map(r => r.m.decisions ?? 0); const R1 = { n: r2.length, rated: rated.length, cleared: r2.filter(r => r.m.cleared).length, dead: mean(r2.map(r => r.m.deadTaps ?? 0)), blind: mean(r2.map(r => r.m.blindRotations ?? 0)), wrong: mean(r2.map(r => r.m.wrongWayRotations ?? 0)), flat: topDiff, decMin: Math.min(...decisions), decMax: Math.max(...decisions), r1n: r1.length, r1blind: mean(r1.map(r => r.m.blindRotations ?? 0)), r1diff: mean(r1.map(r => Number(r.m.rating?.difficulty)).filter(Number.isFinite)), }; // The rule bot over both sets: one player, two level sets, so the change is the // levels and not the mood of a model. const bot = tag => runs.filter(r => String(r.raw.run ?? '').startsWith(tag)); const BOT = tag => { const v = bot(tag); return v.length ? { n: v.length, blind: mean(v.map(r => r.m.blindRotations ?? 0)), dead: mean(v.map(r => r.m.deadTaps ?? 0)), streak: Math.max(...v.map(r => r.m.longestBlindStreak ?? 0)), dec: mean(v.map(r => r.m.decisions ?? 0)), } : null; }; const BOTS = [['第一版', BOT('r1bot-')], ['第二版', BOT('r2bot-')], ['第三版', BOT('r3bot-')]].filter(([, v]) => v); // The rules, each with the observation that forced it. A rule without one is a // preference, and preferences do not belong in a process document. const RULES = [ { n: 1, rule: '难度评分只能按档看,不能逐关看', ev: `${R1.rated} 局里有 ${R1.flat[1]} 局打的是 ${R1.flat[0]} 分,而这些关卡的决策数从 ${R1.decMin} 到 ${R1.decMax}。逐关复测 r = ${f2(retest(r => Number(r.m.rating?.difficulty)).r)},而且 ${ratingDisagreed.length} 对分歧里有 ${disagreedOnSamePlay} 对走的是同一条轨迹——同样的动作、同样的结果,模型给自己打了不同的分。但按档平均约 20 局后,曲线第一次显形了(见上表)。`, why: '文字盘面把每颗棋子的路线、挡路者、紧贴者列成清单,agent 不必用眼睛顺线走一遍——而那正是这游戏的难点。agent 自己写过:「挡路名单已经写全,不用顺着格子数」。剩下的评分噪声,只有靠聚合才压得下去。', now: '逐关排序改为机械假设(旋转深度、棋子数、互锁),由人工审核裁定;评分只用来读整档的高低。' }, { n: 2, rule: '先用规则机器人扫,再花 agent 时间;同一关不要跑两遍', ev: `第二轮意外把每关跑了两次,得到 ${pairs.length} 对复测:其中 ${identical} 对(${pct(identical, pairs.length)})重放了完全相同的动作序列,机械指标复测 r = 1.00。规则机器人跑完 50 关只要 1 秒。`, why: 'agent 近乎确定性,重复同一关几乎不产生新信息。为「提高可信度」而重复跑,是把预算花在已经知道的答案上。', now: '全量验收交给规则机器人;agent 每关只跑一次,省下的预算用来多覆盖关卡。' }, { n: 3, rule: '门槛写在生成期,不写在筛选期', ev: '把开局门槛提到「可点≥3 枚、演出≥0.8s」后筛选写出了 0 关——池子的中位数是 2 枚 / 0.57s。', why: '筛选只能从池子里挑,挑不出池子里没有的东西。分布不对时,要改的是构造。', now: '开局问题由构造里的展示位插入解决,不是靠筛。' }, { n: 4, rule: '每加一个门槛,先证明它挡掉了东西', ev: '两次结构性猜测被实测推翻:「插入端多朝向出口」过滤器对漏斗零效果;为解释难度评分而建的 legibility 指标与评分相关 -0.02。', why: '一个不改变漏斗的门槛,只会让人以为问题已经解决了。', now: '两者都已撤回/降级为纯测量。漏斗每一级的剩余数量都打印出来。' }, { n: 5, rule: '抱怨要对照基线,不要直接当缺陷', ev: `agent 33 条抱怨里最集中的是「路线太长读不完」。但已上线的 v0.5 最长路线 ${v05Longest ?? '—'} 格,比这批新关卡的 ${c50Longest} 格还长。`, why: '那是这个游戏本来的性格。把它当缺陷修掉,改的是游戏而不是关卡。', now: '只在审核页标出,交给人裁定。' }, ]; const html = ` 关卡制作流程复盘

关卡制作流程复盘

这一轮做 50 关的过程里,最有用的结论是一个否定结论:原计划的「agent 打分 → 拟合权重 → 重选关卡」这个闭环合不上,而且不是样本量的问题。下面是它为什么合不上,以及流程该改成什么样。

最新批次 ${esc(LATEST)} · ${R1.n} 局 · 对照批次 ${esc(PREV ?? '—')} ${R1.r1n} 局 · 生成于 ${new Date().toLocaleString('sv-SE').slice(0, 16)}

最新一轮实测到了什么

通关
${R1.cleared}/${R1.n}
无死局
空点 / 关
${f2(R1.dead)}
目标 0
盲转 / 关
${f2(R1.blind)}
目标 0
转错 / 关
${f2(R1.wrong)}
目标 0
难度评分
${R1.flat[1]}/${R1.rated}
都是 ${R1.flat[0]} 分 — 不区分

机械面干净,说明生成期的闸门是有效的:「动了很多次没结果」这类结构缺陷已经被挡在关卡集之外。难度评分那一格是这两轮真正的发现:机械上差着三倍的关卡,它一律给 ${R1.flat[0]} 分。更早的记录里这项的复测可信度是 0.93,那是个更小的样本——在这一批 ${pairs.length} 对复测上实测是 ${f2(retest(r => Number(r.m.rating?.difficulty)).r)},所以它既不区分、也不像以为的那么稳。

两轮之间改了什么,效果如何

第一轮唯一可用的读数是「整批偏易」,加上抱怨里反复出现的「转了没反应」。据此只改了两件事:给连续空转设闸门(最多一次转动没反应,此前有 11 关能连着两次),以及把难度曲线写成旋转深度的配额而不是指望它从排序里自己长出来。第三轮又只改了一件:桶内改按反差取。每轮一个变量,才看得出是哪个变量起的作用。

${BOTS.map(([n]) => ``).join('')} ${BOTS.length ? [ ['盲转 / 关', v => f2(v.blind), true], ['最长连续盲转', v => String(v.streak), true], ['空点 / 关', v => f2(v.dead), false], ['决策 / 关(要做的事)', v => f1(v.dec), false], ].map(([label, get, better]) => `${BOTS.map(([, v], i) => ``).join('')}`).join('\n') : ''}
同一个规则机器人,三套关卡集${n}
${label}${get(v)}
规则机器人数据缺失

要做的事一样多,浪费掉的动作少了一半,而且第三版保住了这个成绩。这是同一个玩家的对照,所以变的是关卡不是模型的心情——一次 50 关的全量验收 1 秒跑完,这也是为什么全量验收该交给它而不是 agent。

${present.map(id => ``).join('')}${present.map(() => '').join('')} ${BANDS.map(([label, a, b]) => `${present.map(id => { const st = bandStats(id, a, b); return ``; }).join('')}`).join('\n')} ${present.map(id => { const st = bandStats(id, 32, 50); return ``; }).join('')}
grok 实测,按档(每关计一次,与它自己玩的那一版对齐)${id}
难度无解感
${label}${f2(st.difficulty)}${f2(st.stuck)}
后档的关卡参数${f1(st.rotations)} 转反差 ${f1(st.contrast)}

第一轮三档几乎是一条平线。第二轮把后档的旋转深度从 ${f1(bandStats(present[0], 32, 50).rotations)} 提到 ${f1(bandStats(present[1] ?? present[0], 32, 50).rotations)} 转,后档评分随之升到 ${f2(bandStats(present[1] ?? present[0], 32, 50).difficulty)}——评分第一次表现出梯度,但只在按档平均之后。

第三轮是一次没被证实的假设。第二轮的数据显示打 3 分的关比打 2 分的关反差高 59%,于是第三轮按反差选关,把后档反差从 ${f1(bandStats(present[1] ?? present[0], 32, 50).contrast)} 提到 ${f1(bandStats(LATEST, 32, 50).contrast)}、旋转深度不变。后档评分没有升,反而是 ${f2(bandStats(LATEST, 32, 50).difficulty)}。以这项评分 r = ${f2(retest(r => Number(r.m.rating?.difficulty)).r)} 的复测可信度,十几关上的这点差距读不出胜负——该说的是「两版分不出来」,不是「反差有用」,也不是「反差没用」。第三版留下来的理由是设计上的:反差是策划点名的好玩机制,而且它的家族分布更均衡(${present.length >= 2 ? '两家族 30/50 → 26/50' : ''}),这两条要靠人工审核裁定。

哪些信号复测得住

下面的 r 是在这一批关卡上现算的:同一个 agent 把每关玩了两次,${pairs.length} 对。不是引用更早、更小的样本。

${[['盲转次数', r => r.m.blindRotations, '可用:驱动了真实修复'], ['转错方向次数', r => r.m.wrongWayRotations, '可用'], ['实际旋转数', r => r.m.rotations, '可用'], ['步数', r => r.m.steps, '可用'], ['决策数', r => r.m.decisions, '可用'], ['无解感评分', r => Number(r.m.rating?.stuck), '只按档看'], ['难度评分', r => Number(r.m.rating?.difficulty), '只按档看,约 20 局起'], ['好玩评分', r => Number(r.m.rating?.fun), '不可用'], ['清晰度评分', r => Number(r.m.rating?.clarity), '不可用']].map(([name, get, use]) => { const t = retest(get); const cls = t.r == null ? '' : t.r >= 0.9 ? 'yes' : t.r >= 0.45 ? '' : 'no'; return ``; }).join('\n')}
信号复测 rn怎么用
${name}${t.r == null ? '—' : f2(t.r)}${t.n}${use}

机械指标全部 r ≈ 1.00,但那多半是因为 agent ${pct(identical, pairs.length)} 的情况下重放了完全相同的动作序列——这本身才是有用的那条结论:重复跑同一关买不到新信息。评分则相反:同一条轨迹,${disagreedOnSamePlay} 对给出了不同的难度分。

五条规则,各自的证据

${RULES.map(r => `

${r.n}${esc(r.rule)}

证据
${esc(r.ev)}
为什么
${esc(r.why)}
现在怎么做
${esc(r.now)}
`).join('\n')}

agent 能测什么,不能测什么

指标可用依据
盲转 / 多余转 / 转错方向可用本批复测 r = ${f2(retest(r => r.m.blindRotations).r)};驱动了真实修复(盲转 1.17 → ${f2(R1.blind)})
是否通关、空点次数可用直接观察,无主观成分
自由文本抱怨可用33 条里两个主题高度重复,指标抓不到;但须对照基线(规则 5)
难度评分只按档看逐关复测 r = ${f2(retest(r => Number(r.m.rating?.difficulty)).r)},且同一条轨迹会打出不同分;按档平均约 20 局才显出梯度
无解感评分只按档看复测 r = ${f2(retest(r => Number(r.m.rating?.stuck)).r)};按档看时与难度同向抬升,可当挫败感的代理
好玩 / 清晰度评分不可用复测 r = ${f2(retest(r => Number(r.m.rating?.fun)).r)} / ${f2(retest(r => Number(r.m.rating?.clarity)).r)},是模型噪声

要做 1000 关的话

  1. 先把判据写成可执行的闸门,再开始生成。这一轮的闸门链是:长成 → 首转双向 → 锁住 ≤90% → 无不可解态 → 终局朝向 ≥2 → 每转释放 ≥2 → 反差 ≥12 → 开局可点 ≥3 → 开局演出 ≥0.9s → 长巡游 ≥3。每一级剩多少必须打印出来。
  2. 用规则机器人做全量验收(12 秒 / 全池),不用 agent。机械旋转指标已证明测的是关卡不是玩家。
  3. agent 只抽样跑,而且只看两样东西:机械缺陷指标,和自由文本抱怨。不要为难度标定花 agent 时间——那条路这一轮已经证明走不通。
  4. 难度曲线靠机械假设铺,靠人眼裁定。用 select-50.mjs 按特征打分并均匀铺开,家族上限防止一套模板占满(上一版 1000 关实际只有 30 套模板)。
  5. 每一轮只动一个变量,并且必须能在漏斗里看到它挡掉了什么,否则撤回。
  6. 人工审核要准备好材料:开局盘面从求解器取(不是示意图)、标明哪些数字可信、每关一个试玩直达链接。

被实测否掉的做法

试过什么结果
拟合 difficulty ~ 机械特征,用来重选关卡响应几乎无方差(${R1.flat[1]}/${R1.rated} 同分),回归的是常数
legibility(一眼看穿的成本)解释难度评分相关 -0.02,无效;降级为纯测量
路线交叠度当难度轴中位已达 89%,没有上升空间
换开局朝向来提高判断成本127 个候选里只有 7 个变好
「插入端多朝向出口」过滤器对漏斗零效果,已删除
把锁住比例当难度主轴排序方向反了:它奖励「开局什么都做不了」
`; writeFileSync(resolve('docs/level-process.html'), html); console.log(`流程复盘已生成:docs/level-process.html(R1 ${R1.n} 关实测,${RULES.length} 条规则)`);