fix(bench): rebuild each round's level set from its runs, not from a snapshot

The round comparison was about to be wrong. levels/campaign-50-r2.json was
copied from levels/campaign-50.json to preserve "the set round 2 played", but
select-50 writes that path in place and several selection experiments ran
between the round and the copy. The file kept 28 of the 50 boards round 2 had
actually played and 46 of round 3's — so the comparison was scoring one round
against another round's levels, and it read as round 3 regressing.

Runs store the whole board and its campaign index, so a round is its own record
and cannot drift. round-set.mjs rebuilds it, the report uses that, and the
snapshot file is gone rather than left lying around looking authoritative.

With the rounds correctly matched, round 3 is not a regression and not an
improvement: back band 2.17 against round 2's 2.39, on a rating that retests at
0.55, which is not enough to separate seventeen levels. The report says that
rather than picking whichever reading flatters the change.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
cabinw
2026-09-14 17:22:31 +08:00
co-authored by Claude Opus 5
parent dcdd4a0ada
commit 4aa0c28cef
9 changed files with 223558 additions and 85313 deletions
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env node
// Rebuild the level set a bench round actually played, from the round's runs.
//
// node scripts/round-set.mjs --round=R2 [--out=…]
//
// Disk snapshots of "the set round N played" are not trustworthy: select-50
// writes levels/campaign-50.json in place, so any selection experiment run
// between the round and the copy silently replaces it. One such copy kept only
// 28 of the 50 boards its round had played, which would have made the round
// comparison compare a set against somebody else's levels.
//
// Runs store the whole board and its campaign index, so the round is its own
// record. That record cannot drift.
import { readdirSync, readFileSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { deepLevelKey } from '../src/deep-lock.js';
const args = Object.fromEntries(process.argv.slice(2).map(a => { const [k, ...v] = a.replace(/^--/, '').split('='); return [k, v.length ? v.join('=') : true]; }));
const ROUND = String(args.round ?? 'R2');
export function roundSet(round, dir = resolve('bench/runs')) {
const byBoard = new Map();
for (const f of readdirSync(dir)) {
if (!f.endsWith('.json') || f === 'index.json') continue;
try {
const d = JSON.parse(readFileSync(resolve(dir, f), 'utf8'));
if (d.status === 'playing' || !d.level?.tiles) continue;
if (!new RegExp(`^${round} · `).test(String(d.agent))) continue;
const k = deepLevelKey(d.level);
if (!byBoard.has(k)) byBoard.set(k, { level: d.level, runs: [] });
byBoard.get(k).runs.push(d);
} catch { /* torn read */ }
}
return [...byBoard.values()].sort((a, b) => (a.level.campaign?.index ?? 0) - (b.level.campaign?.index ?? 0));
}
if (import.meta.url === `file://${process.argv[1]}`) {
const rows = roundSet(ROUND);
const levels = rows.map(r => r.level);
console.log(`${ROUND}:${levels.length} 关,共 ${rows.reduce((s, r) => s + r.runs.length, 0)} 局`);
console.log(`旋转深度 ${JSON.stringify(levels.reduce((m, l) => (m[l.campaign?.rotations ?? '?'] = (m[l.campaign?.rotations ?? '?'] || 0) + 1, m), {}))}`);
if (args.out) { writeFileSync(resolve(args.out), JSON.stringify({ generated: `round-set ${ROUND}`, count: levels.length, levels }, null, 1)); console.log(`已写入 ${args.out}`); }
}