fix(report): key levels by id, not by position in the level set
Regenerating levels/prototypes/d5.json renumbers every level, so the previous build — which resolved runs through the current set's ordinals — silently dropped every run played against an earlier set. The report I committed one commit ago was built that way: 130 runs instead of 229, and the test-retest table down to 7 paired levels from 18. Levels are now keyed by id throughout. Runs carry their own level, so ids the current set no longer contains are read back out of the archive and given an ordinal after the live ones; the cross-model table intersects on ids rather than assuming two sets number the same level alike. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -18,13 +18,18 @@ const RUNS = resolve('bench/runs');
|
||||
// ---------- data ----------
|
||||
const set = JSON.parse(readFileSync(resolve('levels/prototypes/d5.json'), 'utf8'));
|
||||
const levels = Array.isArray(set) ? set : set.levels;
|
||||
const meta = Object.fromEntries(levels.map((l, i) => [l.id, {
|
||||
n: i + 1,
|
||||
// Levels are keyed by id, never by position: regenerating the set renumbers
|
||||
// everything, and keying by ordinal silently dropped every run played against
|
||||
// an earlier set. Runs carry their own level, so ids seen only in the archive
|
||||
// still resolve.
|
||||
const describe = (l, i) => ({
|
||||
n: i == null ? null : i + 1,
|
||||
family: l.id.replace('proto-d5-', '').replace(/-r\d+-s\d+$/, ''),
|
||||
lock: l.prototype ? Math.round(l.prototype.stats.lockedShare * 100) : null,
|
||||
minRot: l.prototype ? l.prototype.stats.rotations : null,
|
||||
cells: l.cells.length, tiles: l.tiles.length,
|
||||
}]));
|
||||
});
|
||||
const meta = Object.fromEntries(levels.map((l, i) => [l.id, describe(l, i)]));
|
||||
|
||||
const BATCHES = [
|
||||
{ tag: 'Grok Build CLI · xhigh', label: 'grok-4.6 · 旧盘面', short: 'grok旧', format: 'old' },
|
||||
@@ -44,12 +49,18 @@ const NEW_TAGS = ['pi · Grok 4.6 xhigh', 'pi · Grok 4.6 xhigh · 复测', 'Kim
|
||||
|
||||
const runs = readdirSync(RUNS).filter(f => f.endsWith('.json') && f !== 'index.json')
|
||||
.map(f => { try { return JSON.parse(readFileSync(resolve(RUNS, f), 'utf8')); } catch { return null; } })
|
||||
.filter(Boolean).filter(d => meta[d.level?.id]);
|
||||
.filter(Boolean).filter(d => d.level?.id?.startsWith('proto-d5-'));
|
||||
// Levels retired by a regeneration still have runs worth comparing.
|
||||
let retired = 0;
|
||||
for (const d of runs) if (!meta[d.level.id]) { meta[d.level.id] = describe(d.level, null); retired++; }
|
||||
// Give every level a stable ordinal: current set first, retired ones after.
|
||||
let ord = levels.length;
|
||||
for (const k of Object.keys(meta)) if (meta[k].n == null) meta[k].n = ++ord;
|
||||
|
||||
const byTag = {};
|
||||
for (const b of BATCHES) {
|
||||
const rows = {};
|
||||
for (const d of runs) if (d.agent === b.tag) rows[meta[d.level.id].n] = { m: metrics(d), lv: meta[d.level.id], run: d.run };
|
||||
for (const d of runs) if (d.agent === b.tag) rows[d.level.id] = { m: metrics(d), lv: meta[d.level.id], run: d.run, id: d.level.id };
|
||||
byTag[b.tag] = rows;
|
||||
}
|
||||
const present = BATCHES.filter(b => Object.keys(byTag[b.tag]).length > 0);
|
||||
@@ -82,7 +93,7 @@ const METRICS = [
|
||||
|
||||
function pair(tagA, tagB) {
|
||||
const A = byTag[tagA] || {}, B = byTag[tagB] || {};
|
||||
const shared = Object.keys(A).filter(k => B[k]).map(Number).sort((a, b) => a - b);
|
||||
const shared = Object.keys(A).filter(k => B[k]).sort((x, y) => meta[x].n - meta[y].n);
|
||||
return { shared, A, B };
|
||||
}
|
||||
|
||||
@@ -184,9 +195,9 @@ const THEMES = [
|
||||
];
|
||||
function notes() {
|
||||
const outp = [];
|
||||
for (const b of present) for (const [L, r] of Object.entries(byTag[b.tag])) {
|
||||
for (const b of present) for (const r of Object.values(byTag[b.tag])) {
|
||||
const w = r.m.rating?.worst; if (!w) continue;
|
||||
outp.push({ batch: b.short, L: Number(L), fam: r.lv.family, text: String(w) });
|
||||
outp.push({ batch: b.short, L: r.lv.n, fam: r.lv.family, text: String(w) });
|
||||
}
|
||||
return outp.sort((a, b) => a.L - b.L);
|
||||
}
|
||||
@@ -203,11 +214,12 @@ const clearRateNote = allCleared
|
||||
const liveTags = NEW_TAGS.filter(t => Object.keys(byTag[t] || {}).length);
|
||||
const triple = m => [m.rotations - m.minimumRotations, m.blindRotations, m.wrongWayRotations].join('/');
|
||||
const crossRows = [];
|
||||
for (let L = 1; L <= levels.length; L++) {
|
||||
const cells = liveTags.map(t => byTag[t][L] ? triple(byTag[t][L].m) : null);
|
||||
if (cells.some(c => c == null)) continue;
|
||||
const lv = Object.values(meta).find(x => x.n === L);
|
||||
crossRows.push({ L, family: lv.family, cells, agree: new Set(cells).size === 1 });
|
||||
const crossIds = [...new Set(liveTags.flatMap(t => Object.keys(byTag[t] || {})))]
|
||||
.filter(id => liveTags.every(t => byTag[t]?.[id]))
|
||||
.sort((a, b) => meta[a].n - meta[b].n);
|
||||
for (const id of crossIds) {
|
||||
const cells = liveTags.map(t => triple(byTag[t][id].m));
|
||||
crossRows.push({ L: meta[id].n, id, family: meta[id].family, cells, agree: new Set(cells).size === 1 });
|
||||
}
|
||||
const agreeN = crossRows.filter(r => r.agree).length;
|
||||
const crossModel = crossRows.length < 3 ? '<p class="sub">同盘面批次不足,跨模型比较尚未成立。</p>' : `
|
||||
|
||||
Reference in New Issue
Block a user