fix(report): key levels by id, not by position in the level set

Regenerating levels/prototypes/d5.json renumbers every level, so the previous
build — which resolved runs through the current set's ordinals — silently
dropped every run played against an earlier set. The report I committed one
commit ago was built that way: 130 runs instead of 229, and the test-retest
table down to 7 paired levels from 18.

Levels are now keyed by id throughout. Runs carry their own level, so ids the
current set no longer contains are read back out of the archive and given an
ordinal after the live ones; the cross-model table intersects on ids rather
than assuming two sets number the same level alike.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
cabinw
2026-09-14 16:04:27 +08:00
co-authored by Claude Opus 5
parent 912bec29b3
commit 8a736766fd
2 changed files with 181 additions and 129 deletions
+25 -13
View File
@@ -18,13 +18,18 @@ const RUNS = resolve('bench/runs');
// ---------- data ----------
const set = JSON.parse(readFileSync(resolve('levels/prototypes/d5.json'), 'utf8'));
const levels = Array.isArray(set) ? set : set.levels;
const meta = Object.fromEntries(levels.map((l, i) => [l.id, {
n: i + 1,
// Levels are keyed by id, never by position: regenerating the set renumbers
// everything, and keying by ordinal silently dropped every run played against
// an earlier set. Runs carry their own level, so ids seen only in the archive
// still resolve.
const describe = (l, i) => ({
n: i == null ? null : i + 1,
family: l.id.replace('proto-d5-', '').replace(/-r\d+-s\d+$/, ''),
lock: l.prototype ? Math.round(l.prototype.stats.lockedShare * 100) : null,
minRot: l.prototype ? l.prototype.stats.rotations : null,
cells: l.cells.length, tiles: l.tiles.length,
}]));
});
const meta = Object.fromEntries(levels.map((l, i) => [l.id, describe(l, i)]));
const BATCHES = [
{ tag: 'Grok Build CLI · xhigh', label: 'grok-4.6 · 旧盘面', short: 'grok旧', format: 'old' },
@@ -44,12 +49,18 @@ const NEW_TAGS = ['pi · Grok 4.6 xhigh', 'pi · Grok 4.6 xhigh · 复测', 'Kim
const runs = readdirSync(RUNS).filter(f => f.endsWith('.json') && f !== 'index.json')
.map(f => { try { return JSON.parse(readFileSync(resolve(RUNS, f), 'utf8')); } catch { return null; } })
.filter(Boolean).filter(d => meta[d.level?.id]);
.filter(Boolean).filter(d => d.level?.id?.startsWith('proto-d5-'));
// Levels retired by a regeneration still have runs worth comparing.
let retired = 0;
for (const d of runs) if (!meta[d.level.id]) { meta[d.level.id] = describe(d.level, null); retired++; }
// Give every level a stable ordinal: current set first, retired ones after.
let ord = levels.length;
for (const k of Object.keys(meta)) if (meta[k].n == null) meta[k].n = ++ord;
const byTag = {};
for (const b of BATCHES) {
const rows = {};
for (const d of runs) if (d.agent === b.tag) rows[meta[d.level.id].n] = { m: metrics(d), lv: meta[d.level.id], run: d.run };
for (const d of runs) if (d.agent === b.tag) rows[d.level.id] = { m: metrics(d), lv: meta[d.level.id], run: d.run, id: d.level.id };
byTag[b.tag] = rows;
}
const present = BATCHES.filter(b => Object.keys(byTag[b.tag]).length > 0);
@@ -82,7 +93,7 @@ const METRICS = [
function pair(tagA, tagB) {
const A = byTag[tagA] || {}, B = byTag[tagB] || {};
const shared = Object.keys(A).filter(k => B[k]).map(Number).sort((a, b) => a - b);
const shared = Object.keys(A).filter(k => B[k]).sort((x, y) => meta[x].n - meta[y].n);
return { shared, A, B };
}
@@ -184,9 +195,9 @@ const THEMES = [
];
function notes() {
const outp = [];
for (const b of present) for (const [L, r] of Object.entries(byTag[b.tag])) {
for (const b of present) for (const r of Object.values(byTag[b.tag])) {
const w = r.m.rating?.worst; if (!w) continue;
outp.push({ batch: b.short, L: Number(L), fam: r.lv.family, text: String(w) });
outp.push({ batch: b.short, L: r.lv.n, fam: r.lv.family, text: String(w) });
}
return outp.sort((a, b) => a.L - b.L);
}
@@ -203,11 +214,12 @@ const clearRateNote = allCleared
const liveTags = NEW_TAGS.filter(t => Object.keys(byTag[t] || {}).length);
const triple = m => [m.rotations - m.minimumRotations, m.blindRotations, m.wrongWayRotations].join('/');
const crossRows = [];
for (let L = 1; L <= levels.length; L++) {
const cells = liveTags.map(t => byTag[t][L] ? triple(byTag[t][L].m) : null);
if (cells.some(c => c == null)) continue;
const lv = Object.values(meta).find(x => x.n === L);
crossRows.push({ L, family: lv.family, cells, agree: new Set(cells).size === 1 });
const crossIds = [...new Set(liveTags.flatMap(t => Object.keys(byTag[t] || {})))]
.filter(id => liveTags.every(t => byTag[t]?.[id]))
.sort((a, b) => meta[a].n - meta[b].n);
for (const id of crossIds) {
const cells = liveTags.map(t => triple(byTag[t][id].m));
crossRows.push({ L: meta[id].n, id, family: meta[id].family, cells, agree: new Set(cells).size === 1 });
}
const agreeN = crossRows.filter(r => r.agree).length;
const crossModel = crossRows.length < 3 ? '<p class="sub">同盘面批次不足,跨模型比较尚未成立。</p>' : `