fix(report): key levels by id, not by position in the level set
Regenerating levels/prototypes/d5.json renumbers every level, so the previous build — which resolved runs through the current set's ordinals — silently dropped every run played against an earlier set. The report I committed one commit ago was built that way: 130 runs instead of 229, and the test-retest table down to 7 paired levels from 18. Levels are now keyed by id throughout. Runs carry their own level, so ids the current set no longer contains are read back out of the archive and given an ordinal after the live ones; the cross-model table intersects on ids rather than assuming two sets number the same level alike. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
cabinw
+156
-116
@@ -103,7 +103,7 @@ footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:v
|
|||||||
<div class="eyebrow">HEX TRAILS · 关卡实验台</div>
|
<div class="eyebrow">HEX TRAILS · 关卡实验台</div>
|
||||||
<h1>Agent 评级里，哪些数字真能拿来做关卡</h1>
|
<h1>Agent 评级里，哪些数字真能拿来做关卡</h1>
|
||||||
<p class="lede">同一批关卡、同一个模型、同一套盘面，跑两遍。能重现的指标才配进流程——这份报告把每个指标逐个检验了一遍，结论和直觉相反。</p>
|
<p class="lede">同一批关卡、同一个模型、同一套盘面，跑两遍。能重现的指标才配进流程——这份报告把每个指标逐个检验了一遍，结论和直觉相反。</p>
|
||||||
<p class="meta mono">生成于 2026-09-14 16:03 GMT+8 · 数据来自 bench/runs 的 130 局 · 关卡集 levels/prototypes/d5.json（18 关）</p>
|
<p class="meta mono">生成于 2026-09-14 16:04 GMT+8 · 数据来自 bench/runs 的 229 局 · 关卡集 levels/prototypes/d5.json（18 关）</p>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
<h2><span class="num">01</span>参与检验的批次</h2>
|
<h2><span class="num">01</span>参与检验的批次</h2>
|
||||||
@@ -112,101 +112,101 @@ footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:v
|
|||||||
<div class="scroll"><table>
|
<div class="scroll"><table>
|
||||||
<thead><tr><th>批次</th><th class="n">关数</th><th class="n">通关</th><th class="n">空点</th><th class="n">擦碰空点</th><th class="n">盲转</th><th class="n">转错</th><th class="n">多转</th><th class="n">单局均</th><th class="n">每决策</th></tr></thead>
|
<thead><tr><th>批次</th><th class="n">关数</th><th class="n">通关</th><th class="n">空点</th><th class="n">擦碰空点</th><th class="n">盲转</th><th class="n">转错</th><th class="n">多转</th><th class="n">单局均</th><th class="n">每决策</th></tr></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr><td>grok-4.6 · 旧盘面</td><td class="n">7</td><td class="n">7/7</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">63 s</td><td class="n">6.8 s</td></tr>
|
<tr><td>grok-4.6 · 旧盘面</td><td class="n">18</td><td class="n">18/18</td><td class="n">4</td><td class="n">4</td><td class="n">21</td><td class="n">8</td><td class="n">22</td><td class="n">83 s</td><td class="n">6.8 s</td></tr>
|
||||||
<tr><td>grok-4.6 · 新盘面 · 第 1 次</td><td class="n">7</td><td class="n">7/7</td><td class="n">2</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">39 s</td><td class="n">4.2 s</td></tr>
|
<tr><td>grok-4.6 · 新盘面 · 第 1 次</td><td class="n">18</td><td class="n">18/18</td><td class="n">2</td><td class="n">1</td><td class="n">21</td><td class="n">8</td><td class="n">22</td><td class="n">55 s</td><td class="n">3.9 s</td></tr>
|
||||||
<tr><td>grok-4.6 · 新盘面 · 第 2 次</td><td class="n">7</td><td class="n">7/7</td><td class="n">0</td><td class="n">0</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">38 s</td><td class="n">3.4 s</td></tr>
|
<tr><td>grok-4.6 · 新盘面 · 第 2 次</td><td class="n">18</td><td class="n">18/18</td><td class="n">0</td><td class="n">0</td><td class="n">25</td><td class="n">10</td><td class="n">28</td><td class="n">49 s</td><td class="n">3.9 s</td></tr>
|
||||||
<tr><td>codex-spark · 旧盘面</td><td class="n">3</td><td class="n">3/3</td><td class="n">0</td><td class="n">0</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">16 s</td><td class="n">1.4 s</td></tr>
|
<tr><td>codex-spark · 旧盘面</td><td class="n">6</td><td class="n">6/6</td><td class="n">0</td><td class="n">0</td><td class="n">2</td><td class="n">2</td><td class="n">2</td><td class="n">15 s</td><td class="n">1.4 s</td></tr>
|
||||||
<tr><td>gpt-6-astra · low · 新盘面</td><td class="n">7</td><td class="n">7/7</td><td class="n">0</td><td class="n">0</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">36 s</td><td class="n">7.9 s</td></tr>
|
<tr><td>gpt-6-astra · low · 新盘面</td><td class="n">18</td><td class="n">18/18</td><td class="n">1</td><td class="n">0</td><td class="n">21</td><td class="n">8</td><td class="n">22</td><td class="n">41 s</td><td class="n">7.0 s</td></tr>
|
||||||
<tr><td>codex-spark · xhigh · 新盘面</td><td class="n">7</td><td class="n">7/7</td><td class="n">0</td><td class="n">0</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">19 s</td><td class="n">1.8 s</td></tr>
|
<tr><td>codex-spark · xhigh · 新盘面</td><td class="n">18</td><td class="n">17/18</td><td class="n">0</td><td class="n">0</td><td class="n">21</td><td class="n">7</td><td class="n">21</td><td class="n">21 s</td><td class="n">1.8 s</td></tr>
|
||||||
<tr><td>kimi-k3 · high · 新盘面</td><td class="n">7</td><td class="n">7/7</td><td class="n">0</td><td class="n">0</td><td class="n">1</td><td class="n">1</td><td class="n">1</td><td class="n">42 s</td><td class="n">5.2 s</td></tr>
|
<tr><td>kimi-k3 · high · 新盘面</td><td class="n">18</td><td class="n">18/18</td><td class="n">0</td><td class="n">0</td><td class="n">21</td><td class="n">8</td><td class="n">22</td><td class="n">48 s</td><td class="n">5.2 s</td></tr>
|
||||||
<tr><td>grok-4.6 · 新关卡集（终局门槛后）</td><td class="n">18</td><td class="n">18/18</td><td class="n">0</td><td class="n">0</td><td class="n">5</td><td class="n">4</td><td class="n">6</td><td class="n">41 s</td><td class="n">3.8 s</td></tr>
|
<tr><td>grok-4.6 · 新关卡集（终局门槛后）</td><td class="n">18</td><td class="n">18/18</td><td class="n">0</td><td class="n">0</td><td class="n">5</td><td class="n">4</td><td class="n">6</td><td class="n">41 s</td><td class="n">3.8 s</td></tr>
|
||||||
</tbody></table></div>
|
</tbody></table></div>
|
||||||
<p class="sub">所有批次通关率都是 100%。<b>这本身就是一条结论：通关率在这批内容上毫无区分度，任何流程都不该建立在通过/失败上。</b></p>
|
<p class="sub">已完成的对局通关率全部是 100%（个别批次仍有对局在进行中，未计入）。<b>通关率在这批内容上毫无区分度，任何流程都不该建立在通过/失败上。</b></p>
|
||||||
|
|
||||||
|
|
||||||
<h2><span class="num">02</span>信度检验：跑两遍还一样吗</h2>
|
<h2><span class="num">02</span>信度检验：跑两遍还一样吗</h2>
|
||||||
<p>同模型（grok-4.6 xhigh）、同 harness（pi）、同盘面（新）、同 7 关，只是又跑了一遍。两次之间唯一的变量就是运行本身，所以下面的相关系数<b>就是这个指标的复测信度</b>。</p>
|
<p>同模型（grok-4.6 xhigh）、同 harness（pi）、同盘面（新）、同 18 关，只是又跑了一遍。两次之间唯一的变量就是运行本身，所以下面的相关系数<b>就是这个指标的复测信度</b>。</p>
|
||||||
<p class="sub">判定规则：<b>关间标准差 < 0.35</b> 说明这个指标在不同关卡上几乎一样，再稳也没用（无法排序）；<b>复测 r < 0.5</b> 说明换一次运行结果就变，不能作为依据。两条都过才算可用。</p>
|
<p class="sub">判定规则：<b>关间标准差 < 0.35</b> 说明这个指标在不同关卡上几乎一样，再稳也没用（无法排序）；<b>复测 r < 0.5</b> 说明换一次运行结果就变，不能作为依据。两条都过才算可用。</p>
|
||||||
<div class="scroll"><table>
|
<div class="scroll"><table>
|
||||||
<thead><tr><th>指标</th><th class="n">关间标准差</th><th class="n">复测相关 r</th><th class="n">逐关完全一致</th><th>判定</th></tr></thead>
|
<thead><tr><th>指标</th><th class="n">关间标准差</th><th class="n">复测相关 r</th><th class="n">逐关完全一致</th><th>判定</th></tr></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr><td>多转<br><small class="sub">清盘旋转数 − 求解器最少旋转数</small></td><td class="n">0.35</td><td class="n">1.00</td><td class="n">7/7</td><td><span class="tag warn">无区分度</span></td></tr>
|
<tr><td>多转<br><small class="sub">清盘旋转数 − 求解器最少旋转数</small></td><td class="n">1.58</td><td class="n">0.87</td><td class="n">17/18</td><td><span class="tag good">可用</span></td></tr>
|
||||||
<tr><td>盲转<br><small class="sub">转了一次，什么出口都没打开</small></td><td class="n">0.35</td><td class="n">1.00</td><td class="n">7/7</td><td><span class="tag warn">无区分度</span></td></tr>
|
<tr><td>盲转<br><small class="sub">转了一次，什么出口都没打开</small></td><td class="n">1.57</td><td class="n">0.92</td><td class="n">17/18</td><td><span class="tag good">可用 · 高</span></td></tr>
|
||||||
<tr><td>转错方向<br><small class="sub">另一边本来能打开</small></td><td class="n">0.35</td><td class="n">1.00</td><td class="n">7/7</td><td><span class="tag warn">无区分度</span></td></tr>
|
<tr><td>转错方向<br><small class="sub">另一边本来能打开</small></td><td class="n">0.50</td><td class="n">0.82</td><td class="n">17/18</td><td><span class="tag good">可用</span></td></tr>
|
||||||
<tr><td>步数</td><td class="n">1.60</td><td class="n">0.92</td><td class="n">6/7</td><td><span class="tag good">可用 · 高</span></td></tr>
|
<tr><td>步数</td><td class="n">2.24</td><td class="n">0.76</td><td class="n">16/18</td><td><span class="tag good">可用</span></td></tr>
|
||||||
<tr><td>决策数<br><small class="sub">看了几次盘面</small></td><td class="n">1.39</td><td class="n">0.84</td><td class="n">3/7</td><td><span class="tag good">可用</span></td></tr>
|
<tr><td>决策数<br><small class="sub">看了几次盘面</small></td><td class="n">1.89</td><td class="n">0.77</td><td class="n">10/18</td><td><span class="tag good">可用</span></td></tr>
|
||||||
<tr><td>空点<br><small class="sub">点了没动</small></td><td class="n">0.70</td><td class="n">—</td><td class="n">6/7</td><td><span class="tag signal">噪声</span></td></tr>
|
<tr><td>空点<br><small class="sub">点了没动</small></td><td class="n">0.46</td><td class="n">—</td><td class="n">17/18</td><td><span class="tag signal">噪声</span></td></tr>
|
||||||
<tr><td>动作÷棋子</td><td class="n">0.09</td><td class="n">0.86</td><td class="n">6/7</td><td><span class="tag warn">无区分度</span></td></tr>
|
<tr><td>动作÷棋子</td><td class="n">0.15</td><td class="n">0.69</td><td class="n">16/18</td><td><span class="tag warn">无区分度</span></td></tr>
|
||||||
<tr><td>评分：无解感</td><td class="n">0.49</td><td class="n">0.09</td><td class="n">4/7</td><td><span class="tag signal">噪声</span></td></tr>
|
<tr><td>评分：无解感</td><td class="n">0.87</td><td class="n">0.85</td><td class="n">14/18</td><td><span class="tag good">可用</span></td></tr>
|
||||||
<tr><td>评分：难度</td><td class="n">0.35</td><td class="n">1.00</td><td class="n">7/7</td><td><span class="tag warn">无区分度</span></td></tr>
|
<tr><td>评分：难度</td><td class="n">0.59</td><td class="n">0.93</td><td class="n">17/18</td><td><span class="tag good">可用 · 高</span></td></tr>
|
||||||
<tr><td>评分：旋转意义</td><td class="n">0.49</td><td class="n">0.09</td><td class="n">4/7</td><td><span class="tag signal">噪声</span></td></tr>
|
<tr><td>评分：旋转意义</td><td class="n">0.37</td><td class="n">0.32</td><td class="n">15/18</td><td><span class="tag signal">噪声</span></td></tr>
|
||||||
<tr><td>评分：清晰度</td><td class="n">0.35</td><td class="n">-0.17</td><td class="n">5/7</td><td><span class="tag warn">无区分度</span></td></tr>
|
<tr><td>评分：清晰度</td><td class="n">0.47</td><td class="n">0.19</td><td class="n">12/18</td><td><span class="tag signal">噪声</span></td></tr>
|
||||||
<tr><td>评分：好玩</td><td class="n">0.35</td><td class="n">—</td><td class="n">6/7</td><td><span class="tag warn">无区分度</span></td></tr>
|
<tr><td>评分：好玩</td><td class="n">0.42</td><td class="n">0.56</td><td class="n">15/18</td><td><span class="tag good">可用</span></td></tr>
|
||||||
</tbody></table></div>
|
</tbody></table></div>
|
||||||
<p class="sub">「关间标准差」衡量这个指标能不能把不同关卡区分开——标准差接近 0 的指标就算完全可复现也没用，因为每关都一样。<b>两个条件要同时满足</b>：既能区分关卡，又能跑两遍不变。</p>
|
<p class="sub">「关间标准差」衡量这个指标能不能把不同关卡区分开——标准差接近 0 的指标就算完全可复现也没用，因为每关都一样。<b>两个条件要同时满足</b>：既能区分关卡，又能跑两遍不变。</p>
|
||||||
|
|
||||||
<div class="scatters">
|
<div class="scatters">
|
||||||
<div class="sc"><h4>多转</h4><div class="r">r = 1.00</div><svg viewBox="0 0 168 168" role="img" aria-label="多转 两次运行散点图，相关系数 1.00">
|
<div class="sc"><h4>多转</h4><div class="r">r = 0.87</div><svg viewBox="0 0 168 168" role="img" aria-label="多转 两次运行散点图，相关系数 0.87">
|
||||||
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
||||||
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
||||||
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
||||||
<g class="dot flat"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="10.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
<g class="dot ok"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="39.2" cy="128.8" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="39.2" cy="128.8" r="3.6"/><circle cx="78.8" cy="10.0" r="3.6"/><circle cx="78.8" cy="89.2" r="3.6"/><circle cx="78.8" cy="89.2" r="3.6"/><circle cx="78.8" cy="89.2" r="3.6"/><circle cx="39.2" cy="128.8" r="3.6"/><circle cx="39.2" cy="128.8" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="52.4" cy="115.6" r="3.6"/></g>
|
||||||
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
||||||
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
||||||
</svg></div>
|
</svg></div>
|
||||||
<div class="sc"><h4>盲转</h4><div class="r">r = 1.00</div><svg viewBox="0 0 168 168" role="img" aria-label="盲转 两次运行散点图，相关系数 1.00">
|
<div class="sc"><h4>盲转</h4><div class="r">r = 0.92</div><svg viewBox="0 0 168 168" role="img" aria-label="盲转 两次运行散点图，相关系数 0.92">
|
||||||
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
||||||
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
||||||
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
||||||
<g class="dot flat"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="10.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
<g class="dot solid"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="42.5" cy="125.5" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="42.5" cy="125.5" r="3.6"/><circle cx="92.0" cy="10.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="42.5" cy="125.5" r="3.6"/><circle cx="42.5" cy="125.5" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="42.5" cy="125.5" r="3.6"/></g>
|
||||||
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
||||||
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
||||||
</svg></div>
|
</svg></div>
|
||||||
<div class="sc"><h4>转错方向</h4><div class="r">r = 1.00</div><svg viewBox="0 0 168 168" role="img" aria-label="转错方向 两次运行散点图，相关系数 1.00">
|
<div class="sc"><h4>转错方向</h4><div class="r">r = 0.82</div><svg viewBox="0 0 168 168" role="img" aria-label="转错方向 两次运行散点图，相关系数 0.82">
|
||||||
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
||||||
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
||||||
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
||||||
<g class="dot flat"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="10.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
<g class="dot ok"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="70.0" cy="10.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
||||||
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
||||||
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
||||||
</svg></div>
|
</svg></div>
|
||||||
<div class="sc"><h4>步数</h4><div class="r">r = 0.92</div><svg viewBox="0 0 168 168" role="img" aria-label="步数 两次运行散点图，相关系数 0.92">
|
<div class="sc"><h4>步数</h4><div class="r">r = 0.76</div><svg viewBox="0 0 168 168" role="img" aria-label="步数 两次运行散点图，相关系数 0.76">
|
||||||
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
||||||
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
||||||
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
||||||
<g class="dot solid"><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="158.0" cy="54.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
<g class="dot ok"><circle cx="46.8" cy="121.2" r="3.6"/><circle cx="67.7" cy="114.2" r="3.6"/><circle cx="46.8" cy="121.2" r="3.6"/><circle cx="46.8" cy="121.2" r="3.6"/><circle cx="46.8" cy="121.2" r="3.6"/><circle cx="46.8" cy="121.2" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="53.8" cy="114.2" r="3.6"/><circle cx="74.6" cy="10.0" r="3.6"/><circle cx="53.8" cy="114.2" r="3.6"/><circle cx="74.6" cy="93.4" r="3.6"/><circle cx="74.6" cy="93.4" r="3.6"/><circle cx="53.8" cy="114.2" r="3.6"/><circle cx="32.9" cy="135.1" r="3.6"/><circle cx="46.8" cy="121.2" r="3.6"/><circle cx="60.7" cy="107.3" r="3.6"/></g>
|
||||||
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
||||||
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
||||||
</svg></div>
|
</svg></div>
|
||||||
<div class="sc"><h4>决策数</h4><div class="r">r = 0.84</div><svg viewBox="0 0 168 168" role="img" aria-label="决策数 两次运行散点图，相关系数 0.84">
|
<div class="sc"><h4>决策数</h4><div class="r">r = 0.77</div><svg viewBox="0 0 168 168" role="img" aria-label="决策数 两次运行散点图，相关系数 0.77">
|
||||||
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
||||||
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
||||||
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
||||||
<g class="dot ok"><circle cx="59.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="43.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="109.0" r="3.6"/><circle cx="26.0" cy="109.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
<g class="dot ok"><circle cx="35.4" cy="142.0" r="3.6"/><circle cx="63.7" cy="113.7" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="132.6" r="3.6"/><circle cx="26.0" cy="132.6" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="132.6" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="35.4" cy="123.1" r="3.6"/><circle cx="73.1" cy="10.0" r="3.6"/><circle cx="63.7" cy="104.3" r="3.6"/><circle cx="63.7" cy="104.3" r="3.6"/><circle cx="73.1" cy="94.9" r="3.6"/><circle cx="35.4" cy="132.6" r="3.6"/><circle cx="35.4" cy="132.6" r="3.6"/><circle cx="26.0" cy="123.1" r="3.6"/><circle cx="54.3" cy="113.7" r="3.6"/></g>
|
||||||
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
||||||
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
||||||
</svg></div>
|
</svg></div>
|
||||||
<div class="sc"><h4>动作÷棋子</h4><div class="r">r = 0.86</div><svg viewBox="0 0 168 168" role="img" aria-label="动作÷棋子 两次运行散点图，相关系数 0.86">
|
<div class="sc"><h4>动作÷棋子</h4><div class="r">r = 0.69</div><svg viewBox="0 0 168 168" role="img" aria-label="动作÷棋子 两次运行散点图，相关系数 0.69">
|
||||||
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
||||||
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
||||||
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
||||||
<g class="dot flat"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="98.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/></g>
|
<g class="dot flat"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="50.8" cy="133.8" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="34.2" cy="133.8" r="3.6"/><circle cx="34.2" cy="133.8" r="3.6"/><circle cx="34.2" cy="133.8" r="3.6"/><circle cx="34.2" cy="133.8" r="3.6"/><circle cx="59.0" cy="10.0" r="3.6"/><circle cx="78.2" cy="89.8" r="3.6"/><circle cx="59.0" cy="109.0" r="3.6"/><circle cx="59.0" cy="109.0" r="3.6"/><circle cx="34.2" cy="133.8" r="3.6"/><circle cx="45.2" cy="122.8" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="42.5" cy="125.5" r="3.6"/></g>
|
||||||
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
||||||
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
||||||
</svg></div>
|
</svg></div>
|
||||||
<div class="sc"><h4>评分：无解感</h4><div class="r">r = 0.09</div><svg viewBox="0 0 168 168" role="img" aria-label="评分：无解感 两次运行散点图，相关系数 0.09">
|
<div class="sc"><h4>评分：无解感</h4><div class="r">r = 0.85</div><svg viewBox="0 0 168 168" role="img" aria-label="评分：无解感 两次运行散点图，相关系数 0.85">
|
||||||
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
||||||
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
||||||
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
||||||
<g class="dot noise"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="10.0" r="3.6"/><circle cx="158.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="10.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
<g class="dot ok"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="70.0" cy="142.0" r="3.6"/><circle cx="70.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="98.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="158.0" cy="54.0" r="3.6"/><circle cx="114.0" cy="54.0" r="3.6"/><circle cx="114.0" cy="54.0" r="3.6"/><circle cx="114.0" cy="54.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="70.0" cy="98.0" r="3.6"/></g>
|
||||||
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
||||||
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
||||||
</svg></div>
|
</svg></div>
|
||||||
<div class="sc"><h4>评分：难度</h4><div class="r">r = 1.00</div><svg viewBox="0 0 168 168" role="img" aria-label="评分：难度 两次运行散点图，相关系数 1.00">
|
<div class="sc"><h4>评分：难度</h4><div class="r">r = 0.93</div><svg viewBox="0 0 168 168" role="img" aria-label="评分：难度 两次运行散点图，相关系数 0.93">
|
||||||
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
<line class="ax" x1="26" y1="142" x2="160" y2="142"/>
|
||||||
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
<line class="ax" x1="26" y1="8" x2="26" y2="142"/>
|
||||||
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
<line class="diag" x1="26" y1="142" x2="158" y2="10"/>
|
||||||
<g class="dot flat"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="10.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
<g class="dot solid"><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="158.0" cy="10.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="92.0" cy="76.0" r="3.6"/><circle cx="26.0" cy="76.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/><circle cx="26.0" cy="142.0" r="3.6"/></g>
|
||||||
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
<text class="axl" x="84" y="162" text-anchor="middle">第 1 次</text>
|
||||||
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
<text class="axl" x="10" y="84" text-anchor="middle" transform="rotate(-90 10 84)">第 2 次</text>
|
||||||
</svg></div>
|
</svg></div>
|
||||||
@@ -220,18 +220,18 @@ footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:v
|
|||||||
<div class="scroll"><table>
|
<div class="scroll"><table>
|
||||||
<thead><tr><th>指标</th><th class="n">控制变量后 r</th><th class="n">换盘面后 r</th><th>差异说明</th></tr></thead>
|
<thead><tr><th>指标</th><th class="n">控制变量后 r</th><th class="n">换盘面后 r</th><th>差异说明</th></tr></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr><td>多转</td><td class="n">1.00</td><td class="n">1.00</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
<tr><td>多转</td><td class="n">0.87</td><td class="n">1.00</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
||||||
<tr><td>盲转</td><td class="n">1.00</td><td class="n">1.00</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
<tr><td>盲转</td><td class="n">0.92</td><td class="n">1.00</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
||||||
<tr><td>转错方向</td><td class="n">1.00</td><td class="n">1.00</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
<tr><td>转错方向</td><td class="n">0.82</td><td class="n">1.00</td><td class="sub">控制变量后反而更低——值得留意</td></tr>
|
||||||
<tr><td>步数</td><td class="n">0.92</td><td class="n">0.86</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
<tr><td>步数</td><td class="n">0.76</td><td class="n">0.95</td><td class="sub">控制变量后反而更低——值得留意</td></tr>
|
||||||
<tr><td>决策数</td><td class="n">0.84</td><td class="n">0.33</td><td class="sub">控制变量后明显更高——之前被盘面变化压低了</td></tr>
|
<tr><td>决策数</td><td class="n">0.77</td><td class="n">0.87</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
||||||
<tr><td>空点</td><td class="n">—</td><td class="n">-0.17</td><td class="sub">—</td></tr>
|
<tr><td>空点</td><td class="n">—</td><td class="n">-0.10</td><td class="sub">—</td></tr>
|
||||||
<tr><td>动作÷棋子</td><td class="n">0.86</td><td class="n">0.63</td><td class="sub">控制变量后明显更高——之前被盘面变化压低了</td></tr>
|
<tr><td>动作÷棋子</td><td class="n">0.69</td><td class="n">0.92</td><td class="sub">控制变量后反而更低——值得留意</td></tr>
|
||||||
<tr><td>评分：无解感</td><td class="n">0.09</td><td class="n">0.42</td><td class="sub">控制变量后反而更低——值得留意</td></tr>
|
<tr><td>评分：无解感</td><td class="n">0.85</td><td class="n">0.80</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
||||||
<tr><td>评分：难度</td><td class="n">1.00</td><td class="n">0.47</td><td class="sub">控制变量后明显更高——之前被盘面变化压低了</td></tr>
|
<tr><td>评分：难度</td><td class="n">0.93</td><td class="n">0.63</td><td class="sub">控制变量后明显更高——之前被盘面变化压低了</td></tr>
|
||||||
<tr><td>评分：旋转意义</td><td class="n">0.09</td><td class="n">-0.17</td><td class="sub">控制变量后明显更高——之前被盘面变化压低了</td></tr>
|
<tr><td>评分：旋转意义</td><td class="n">0.32</td><td class="n">0.06</td><td class="sub">控制变量后明显更高——之前被盘面变化压低了</td></tr>
|
||||||
<tr><td>评分：清晰度</td><td class="n">-0.17</td><td class="n">0.65</td><td class="sub">控制变量后反而更低——值得留意</td></tr>
|
<tr><td>评分：清晰度</td><td class="n">0.19</td><td class="n">0.09</td><td class="sub">两者接近，盘面变化影响很小</td></tr>
|
||||||
<tr><td>评分：好玩</td><td class="n">—</td><td class="n">-0.26</td><td class="sub">—</td></tr>
|
<tr><td>评分：好玩</td><td class="n">0.56</td><td class="n">0.21</td><td class="sub">控制变量后明显更高——之前被盘面变化压低了</td></tr>
|
||||||
</tbody></table></div>
|
</tbody></table></div>
|
||||||
|
|
||||||
|
|
||||||
@@ -239,7 +239,7 @@ footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:v
|
|||||||
|
|
||||||
<p>下表是同一套盘面下所有批次的 <code>多转 / 盲转 / 转错方向</code>。不同厂商的模型、不同 harness、各自独立跑——如果这些数字真的由关卡决定，它们就该一样。</p>
|
<p>下表是同一套盘面下所有批次的 <code>多转 / 盲转 / 转错方向</code>。不同厂商的模型、不同 harness、各自独立跑——如果这些数字真的由关卡决定，它们就该一样。</p>
|
||||||
<div class="grid">
|
<div class="grid">
|
||||||
<div class="tile"><div class="k">逐关完全一致</div><div class="v">7 / 7</div><div class="n">三项数字一个不差</div></div>
|
<div class="tile"><div class="k">逐关完全一致</div><div class="v">16 / 18</div><div class="n">三项数字一个不差</div></div>
|
||||||
<div class="tile"><div class="k">参与批次</div><div class="v">5</div><div class="n">grok新① · grok新② · kimi · astra · spark新</div></div>
|
<div class="tile"><div class="k">参与批次</div><div class="v">5</div><div class="n">grok新① · grok新② · kimi · astra · spark新</div></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="scroll"><table>
|
<div class="scroll"><table>
|
||||||
@@ -252,115 +252,155 @@ footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:v
|
|||||||
<tr><td>L11</td><td>petals</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
<tr><td>L11</td><td>petals</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
||||||
<tr><td>L15</td><td>crossing</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
<tr><td>L15</td><td>crossing</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
||||||
<tr><td>L17</td><td>orbits</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
<tr><td>L17</td><td>orbits</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L19</td><td>petals</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L20</td><td>petals</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L21</td><td>switchback</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L22</td><td>orbits</td><td class="n">4/4/1</td><td class="n">10/8/3</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td><span class="tag signal">✘</span></td></tr>
|
||||||
|
<tr><td>L23</td><td>orbits</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L24</td><td>spiral</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L25</td><td>spiral</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">4/4/1</td><td class="n">3/4/0</td><td><span class="tag signal">✘</span></td></tr>
|
||||||
|
<tr><td>L26</td><td>switchback</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L27</td><td>weave</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td class="n">1/1/1</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L28</td><td>weave</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td class="n">0/0/0</td><td><span class="tag good">✔</span></td></tr>
|
||||||
|
<tr><td>L29</td><td>weave</td><td class="n">2/1/0</td><td class="n">2/1/0</td><td class="n">2/1/0</td><td class="n">2/1/0</td><td class="n">2/1/0</td><td><span class="tag good">✔</span></td></tr>
|
||||||
</tbody></table></div>
|
</tbody></table></div>
|
||||||
<div class="callout good">
|
<div class="callout good">
|
||||||
<p><b>这是全份报告里最硬的一条结论。</b>7/7 关上，独立运行的不同模型给出了完全相同的旋转三元组。这意味着<b>规则机器人 12 秒扫出来的这三个数，等价于大模型跑几小时的结果</b>——这个维度完全不需要 agent。</p>
|
<p><b>这是全份报告里最硬的一条结论。</b>16/18 关上，独立运行的不同模型给出了完全相同的旋转三元组。这意味着<b>规则机器人 12 秒扫出来的这三个数，等价于大模型跑几小时的结果</b>——这个维度完全不需要 agent。</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<h3>不一致的关：不是难，是不稳定</h3>
|
||||||
|
<p class="sub">下面这些关在不同运行之间给出了不同的数字。排除模型差异后，剩下的解释是<b>同一关的体验取决于起手方向</b>——两个玩家玩同一关，一个觉得还行，一个觉得这关有毛病。这比"稳定地难"更值得处理。</p>
|
||||||
|
<div class="note"><b>L22 orbits</b>：4/4/1 ／ 10/8/3 ／ 4/4/1 ／ 4/4/1 ／ 4/4/1</div><div class="note"><b>L25 spiral</b>：4/4/1 ／ 4/4/1 ／ 4/4/1 ／ 4/4/1 ／ 3/4/0</div>
|
||||||
|
|
||||||
<h2><span class="num">04</span>家族结构：机械指标指向同一个地方</h2>
|
<h2><span class="num">04</span>家族结构：机械指标指向同一个地方</h2>
|
||||||
<p>把「多转」按路线家族平均。这是关卡结构问题，不是玩家问题——两批 grok、以及 spark，在这一点上给出一致的结果。</p>
|
<p>把「多转」按路线家族平均。这是关卡结构问题，不是玩家问题——两批 grok、以及 spark，在这一点上给出一致的结果。</p>
|
||||||
|
|
||||||
<div class="card"><h3 style="margin-top:0">grok-4.6 · 旧盘面 · 各家族平均多转</h3><div class="bars">
|
<div class="card"><h3 style="margin-top:0">grok-4.6 · 旧盘面 · 各家族平均多转</h3><div class="bars">
|
||||||
<div class="bar-row"><span class="bar-k">switchback<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">spiral<small>2 关</small></span>
|
||||||
<span class="bar-track"><i style="width:50.0%" class=""></i></span>
|
<span class="bar-track"><i style="width:100.0%" class=""></i></span>
|
||||||
<b class="bar-v">0.5</b></div>
|
<b class="bar-v">4.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">weave<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">orbits<small>3 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:66.7%" class=""></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">2.7</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">petals<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">switchback<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:18.8%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">weave<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:18.8%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">petals<small>4 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">0.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
|
||||||
<div class="bar-row"><span class="bar-k">orbits<small>1 关</small></span>
|
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
|
||||||
<b class="bar-v">0.0</b></div></div></div>
|
<b class="bar-v">0.0</b></div></div></div>
|
||||||
|
|
||||||
<div class="card"><h3 style="margin-top:0">grok-4.6 · 新盘面 · 第 1 次 · 各家族平均多转</h3><div class="bars">
|
<div class="card"><h3 style="margin-top:0">grok-4.6 · 新盘面 · 第 1 次 · 各家族平均多转</h3><div class="bars">
|
||||||
<div class="bar-row"><span class="bar-k">switchback<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">spiral<small>2 关</small></span>
|
||||||
<span class="bar-track"><i style="width:50.0%" class=""></i></span>
|
<span class="bar-track"><i style="width:100.0%" class=""></i></span>
|
||||||
<b class="bar-v">0.5</b></div>
|
<b class="bar-v">4.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">weave<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">orbits<small>3 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:66.7%" class=""></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">2.7</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">petals<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">switchback<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:18.8%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">weave<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:18.8%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">petals<small>4 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">0.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
|
||||||
<div class="bar-row"><span class="bar-k">orbits<small>1 关</small></span>
|
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
|
||||||
<b class="bar-v">0.0</b></div></div></div>
|
<b class="bar-v">0.0</b></div></div></div>
|
||||||
|
|
||||||
<div class="card"><h3 style="margin-top:0">grok-4.6 · 新盘面 · 第 2 次 · 各家族平均多转</h3><div class="bars">
|
<div class="card"><h3 style="margin-top:0">grok-4.6 · 新盘面 · 第 2 次 · 各家族平均多转</h3><div class="bars">
|
||||||
<div class="bar-row"><span class="bar-k">switchback<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">orbits<small>3 关</small></span>
|
||||||
<span class="bar-track"><i style="width:50.0%" class=""></i></span>
|
<span class="bar-track"><i style="width:100.0%" class=""></i></span>
|
||||||
<b class="bar-v">0.5</b></div>
|
<b class="bar-v">4.7</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">weave<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">spiral<small>2 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:85.7%" class=""></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">4.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">petals<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">switchback<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:16.1%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">weave<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:16.1%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">petals<small>4 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">0.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
|
||||||
<div class="bar-row"><span class="bar-k">orbits<small>1 关</small></span>
|
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
|
||||||
<b class="bar-v">0.0</b></div></div></div>
|
<b class="bar-v">0.0</b></div></div></div>
|
||||||
|
|
||||||
|
<div class="card"><h3 style="margin-top:0">codex-spark · 旧盘面 · 各家族平均多转</h3><div class="bars">
|
||||||
|
<div class="bar-row"><span class="bar-k">switchback<small>3 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:33.3%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.3</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">weave<small>3 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:33.3%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.3</b></div></div></div>
|
||||||
|
|
||||||
<div class="card"><h3 style="margin-top:0">gpt-6-astra · low · 新盘面 · 各家族平均多转</h3><div class="bars">
|
<div class="card"><h3 style="margin-top:0">gpt-6-astra · low · 新盘面 · 各家族平均多转</h3><div class="bars">
|
||||||
<div class="bar-row"><span class="bar-k">switchback<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">spiral<small>2 关</small></span>
|
||||||
<span class="bar-track"><i style="width:50.0%" class=""></i></span>
|
<span class="bar-track"><i style="width:100.0%" class=""></i></span>
|
||||||
<b class="bar-v">0.5</b></div>
|
<b class="bar-v">4.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">weave<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">orbits<small>3 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:66.7%" class=""></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">2.7</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">petals<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">switchback<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:18.8%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">weave<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:18.8%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">petals<small>4 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">0.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
|
||||||
<div class="bar-row"><span class="bar-k">orbits<small>1 关</small></span>
|
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
|
||||||
<b class="bar-v">0.0</b></div></div></div>
|
<b class="bar-v">0.0</b></div></div></div>
|
||||||
|
|
||||||
<div class="card"><h3 style="margin-top:0">codex-spark · xhigh · 新盘面 · 各家族平均多转</h3><div class="bars">
|
<div class="card"><h3 style="margin-top:0">codex-spark · xhigh · 新盘面 · 各家族平均多转</h3><div class="bars">
|
||||||
<div class="bar-row"><span class="bar-k">switchback<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">spiral<small>2 关</small></span>
|
||||||
<span class="bar-track"><i style="width:50.0%" class=""></i></span>
|
<span class="bar-track"><i style="width:100.0%" class=""></i></span>
|
||||||
<b class="bar-v">0.5</b></div>
|
<b class="bar-v">3.5</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">weave<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">orbits<small>3 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:76.2%" class=""></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">2.7</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">petals<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">switchback<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:21.4%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">weave<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:21.4%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">petals<small>4 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">0.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
|
||||||
<div class="bar-row"><span class="bar-k">orbits<small>1 关</small></span>
|
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
|
||||||
<b class="bar-v">0.0</b></div></div></div>
|
<b class="bar-v">0.0</b></div></div></div>
|
||||||
|
|
||||||
<div class="card"><h3 style="margin-top:0">kimi-k3 · high · 新盘面 · 各家族平均多转</h3><div class="bars">
|
<div class="card"><h3 style="margin-top:0">kimi-k3 · high · 新盘面 · 各家族平均多转</h3><div class="bars">
|
||||||
<div class="bar-row"><span class="bar-k">switchback<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">spiral<small>2 关</small></span>
|
||||||
<span class="bar-track"><i style="width:50.0%" class=""></i></span>
|
<span class="bar-track"><i style="width:100.0%" class=""></i></span>
|
||||||
<b class="bar-v">0.5</b></div>
|
<b class="bar-v">4.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">weave<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">orbits<small>3 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:66.7%" class=""></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">2.7</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">petals<small>2 关</small></span>
|
<div class="bar-row"><span class="bar-k">switchback<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:18.8%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">weave<small>4 关</small></span>
|
||||||
|
<span class="bar-track"><i style="width:18.8%" class=""></i></span>
|
||||||
|
<b class="bar-v">0.8</b></div>
|
||||||
|
<div class="bar-row"><span class="bar-k">petals<small>4 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
<b class="bar-v">0.0</b></div>
|
||||||
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
<div class="bar-row"><span class="bar-k">crossing<small>1 关</small></span>
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
||||||
<b class="bar-v">0.0</b></div>
|
|
||||||
<div class="bar-row"><span class="bar-k">orbits<small>1 关</small></span>
|
|
||||||
<span class="bar-track"><i style="width:0.0%" class="zero"></i></span>
|
|
||||||
<b class="bar-v">0.0</b></div></div></div>
|
<b class="bar-v">0.0</b></div></div></div>
|
||||||
|
|
||||||
<div class="card"><h3 style="margin-top:0">grok-4.6 · 新关卡集（终局门槛后） · 各家族平均多转</h3><div class="bars">
|
<div class="card"><h3 style="margin-top:0">grok-4.6 · 新关卡集（终局门槛后） · 各家族平均多转</h3><div class="bars">
|
||||||
@@ -386,19 +426,19 @@ footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:v
|
|||||||
<h2><span class="num">05</span>Agent 说了什么</h2>
|
<h2><span class="num">05</span>Agent 说了什么</h2>
|
||||||
<p class="sub">评分是 1–5 的数字，<code>worst</code> 是它自己写的一句话。下面按反复出现的主题归类——<b>价值在重复模式，不在单条内容</b>。</p>
|
<p class="sub">评分是 1–5 的数字，<code>worst</code> 是它自己写的一句话。下面按反复出现的主题归类——<b>价值在重复模式，不在单条内容</b>。</p>
|
||||||
|
|
||||||
<h3>擦碰看不出来 · 13 次提及</h3>
|
<h3>擦碰看不出来 · 30 次提及</h3>
|
||||||
<div class="note"><b>L3 switchback</b>（grok旧）：开局 t2/t11 路上看着没人其实会擦到，差点空点；只剩 t0 和 t3 对堵时短暂以为卡住了</div>
|
<div class="note"><b>L3 switchback</b>（grok旧）：开局 t2/t11 路上看着没人其实会擦到，差点空点；只剩 t0 和 t3 对堵时短暂以为卡住了</div>
|
||||||
<div class="note"><b>L6 weave</b>（grok旧）：t5走后（第9步）t2和t3都像能出界，其实互相擦碰，只能再转</div>
|
<div class="note"><b>L6 weave</b>（grok旧）：t5走后（第9步）t2和t3都像能出界，其实互相擦碰，只能再转</div>
|
||||||
<div class="note"><b>L6 weave</b>（grok新①）：朝向1时 t2 和 t3 互相擦碰、t0 被 t2 压着，短暂不确定是再转还是还有能点的</div>
|
<div class="note"><b>L6 weave</b>（grok新①）：朝向1时 t2 和 t3 互相擦碰、t0 被 t2 压着，短暂不确定是再转还是还有能点的</div>
|
||||||
<div class="note"><b>L6 weave</b>（grok新②）：第9步清完后t2和t3互相擦挡，谁先走都不行，只能转</div>
|
<div class="note"><b>L6 weave</b>（grok新②）：第9步清完后t2和t3互相擦挡，谁先走都不行，只能转</div>
|
||||||
|
|
||||||
<h3>不知道该转哪边 · 9 次提及</h3>
|
<h3>不知道该转哪边 · 19 次提及</h3>
|
||||||
<div class="note"><b>L1 switchback</b>（grok新关）：朝向3右转后没有新出口，t0和t3仍互相挡，短暂怀疑该不该改左转</div>
|
<div class="note"><b>L1 switchback</b>（grok新关）：朝向3右转后没有新出口，t0和t3仍互相挡，短暂怀疑该不该改左转</div>
|
||||||
<div class="note"><b>L5 switchback</b>（grok旧）：第 13 步右转后朝向 3 没新出口，t0 和 t3 仍互相挡，不确定该不该继续同向转</div>
|
<div class="note"><b>L5 switchback</b>（grok旧）：第 13 步右转后朝向 3 没新出口，t0 和 t3 仍互相挡，不确定该不该继续同向转</div>
|
||||||
<div class="note"><b>L5 switchback</b>（grok新①）：第15步右转后朝向3没有新出口，最后两枚还互压，一度担心转错了方向</div>
|
<div class="note"><b>L5 switchback</b>（grok新①）：第15步右转后朝向3没有新出口，最后两枚还互压，一度担心转错了方向</div>
|
||||||
<div class="note"><b>L5 switchback</b>（spark旧）：第 8-9 步清掉 t5,t4,t2 后只剩 t0/t3 互压，朝向2不再有出口，开始不确定该是继续走还是转哪边</div>
|
<div class="note"><b>L5 switchback</b>（spark旧）：第 8-9 步清掉 t5,t4,t2 后只剩 t0/t3 互压，朝向2不再有出口，开始不确定该是继续走还是转哪边</div>
|
||||||
|
|
||||||
<h3>长路线读不动 · 25 次提及</h3>
|
<h3>长路线读不动 · 32 次提及</h3>
|
||||||
<div class="note"><b>L2 weave</b>（grok新关）：开局 t0 绕了 21 格还压着 6 枚，一眼看不出清障顺序，只能先点边上的 t4/t6</div>
|
<div class="note"><b>L2 weave</b>（grok新关）：开局 t0 绕了 21 格还压着 6 枚，一眼看不出清障顺序，只能先点边上的 t4/t6</div>
|
||||||
<div class="note"><b>L3 switchback</b>（kimi）：第 10 步后 t0/t2/t3/t10 四枚互相压成环，盘面上看不出解法，只能靠旋转试出新朝向</div>
|
<div class="note"><b>L3 switchback</b>（kimi）：第 10 步后 t0/t2/t3/t10 四枚互相压成环，盘面上看不出解法，只能靠旋转试出新朝向</div>
|
||||||
<div class="note"><b>L9 petals</b>（grok新关）：开局 t0/t3 各绕 19 格互相穿过，一时看不出该先剥哪层</div>
|
<div class="note"><b>L9 petals</b>（grok新关）：开局 t0/t3 各绕 19 格互相穿过，一时看不出该先剥哪层</div>
|
||||||
@@ -409,7 +449,7 @@ footer{margin-top:64px;padding-top:22px;border-top:1px solid var(--rule);color:v
|
|||||||
<ol class="steps">
|
<ol class="steps">
|
||||||
<li><h3>生成期硬门槛 · 零成本</h3><p class="sub">用已有求解器，在生成时就卡掉：最少旋转 ≥ 2、开局锁住率 ≤ 90%、首转两个方向都能推进、无死局状态。不合格的根本不进候选池。</p></li>
|
<li><h3>生成期硬门槛 · 零成本</h3><p class="sub">用已有求解器，在生成时就卡掉：最少旋转 ≥ 2、开局锁住率 ≤ 90%、首转两个方向都能推进、无死局状态。不合格的根本不进候选池。</p></li>
|
||||||
<li><h3>规则机器人全量扫 · 1000 关 12 秒</h3><p class="sub">产出盲转 / 多转 / 转错方向。按 <code>盲转 ≥ 2 或 多转 ≥ 2</code> 打标。这条规则在 D5 上正好命中 orbits+spiral 那几关，和两个大模型跑两小时得到的结论一致。</p></li>
|
<li><h3>规则机器人全量扫 · 1000 关 12 秒</h3><p class="sub">产出盲转 / 多转 / 转错方向。按 <code>盲转 ≥ 2 或 多转 ≥ 2</code> 打标。这条规则在 D5 上正好命中 orbits+spiral 那几关，和两个大模型跑两小时得到的结论一致。</p></li>
|
||||||
<li><h3>Agent 只跑被标记的 + 5% 随机抽样</h3><p class="sub">不是 1000 关全跑。收集<b>三样东西</b>：<code>difficulty</code>（r=0.93，可用来给 1000 关排序）、<code>stuck</code>（r=0.85，体感无解的直接信号）、以及 <code>worst</code> 文字（重复模式能挖出系统性缺陷）。<b>丢掉 rotation / clarity / fun</b>——复测 r 只有 0.32 / 0.19 / 0.56，是模型噪声。按实测 36 s/关、4 路并行，150 关约半小时。</p></li>
|
<li><h3>Agent 只跑被标记的 + 5% 随机抽样</h3><p class="sub">不是 1000 关全跑。收集<b>三样东西</b>：<code>difficulty</code>（r=0.93，可用来给 1000 关排序）、<code>stuck</code>（r=0.85，体感无解的直接信号）、以及 <code>worst</code> 文字（重复模式能挖出系统性缺陷）。<b>丢掉 rotation / clarity / fun</b>——复测 r 只有 0.32 / 0.19 / 0.56，是模型噪声。按实测 43 s/关、4 路并行，150 关约半小时。</p></li>
|
||||||
<li><h3>真人标定锚点 · 5–10 关</h3><p class="sub">Agent 的难度<b>排序</b>可信（复测 r=0.93），但绝对值可能整体偏移——它觉得「刚好」的关，真人可能觉得太简单。所以真人只需要玩少数几关当<b>锚点</b>，把 agent 的相对排序映射到真实难度刻度上，不必逐关评。这比原本设想的省很多人力。</p></li>
|
<li><h3>真人标定锚点 · 5–10 关</h3><p class="sub">Agent 的难度<b>排序</b>可信（复测 r=0.93），但绝对值可能整体偏移——它觉得「刚好」的关，真人可能觉得太简单。所以真人只需要玩少数几关当<b>锚点</b>，把 agent 的相对排序映射到真实难度刻度上，不必逐关评。这比原本设想的省很多人力。</p></li>
|
||||||
</ol>
|
</ol>
|
||||||
<div class="callout good">
|
<div class="callout good">
|
||||||
|
|||||||
@@ -18,13 +18,18 @@ const RUNS = resolve('bench/runs');
|
|||||||
// ---------- data ----------
|
// ---------- data ----------
|
||||||
const set = JSON.parse(readFileSync(resolve('levels/prototypes/d5.json'), 'utf8'));
|
const set = JSON.parse(readFileSync(resolve('levels/prototypes/d5.json'), 'utf8'));
|
||||||
const levels = Array.isArray(set) ? set : set.levels;
|
const levels = Array.isArray(set) ? set : set.levels;
|
||||||
const meta = Object.fromEntries(levels.map((l, i) => [l.id, {
|
// Levels are keyed by id, never by position: regenerating the set renumbers
|
||||||
n: i + 1,
|
// everything, and keying by ordinal silently dropped every run played against
|
||||||
|
// an earlier set. Runs carry their own level, so ids seen only in the archive
|
||||||
|
// still resolve.
|
||||||
|
const describe = (l, i) => ({
|
||||||
|
n: i == null ? null : i + 1,
|
||||||
family: l.id.replace('proto-d5-', '').replace(/-r\d+-s\d+$/, ''),
|
family: l.id.replace('proto-d5-', '').replace(/-r\d+-s\d+$/, ''),
|
||||||
lock: l.prototype ? Math.round(l.prototype.stats.lockedShare * 100) : null,
|
lock: l.prototype ? Math.round(l.prototype.stats.lockedShare * 100) : null,
|
||||||
minRot: l.prototype ? l.prototype.stats.rotations : null,
|
minRot: l.prototype ? l.prototype.stats.rotations : null,
|
||||||
cells: l.cells.length, tiles: l.tiles.length,
|
cells: l.cells.length, tiles: l.tiles.length,
|
||||||
}]));
|
});
|
||||||
|
const meta = Object.fromEntries(levels.map((l, i) => [l.id, describe(l, i)]));
|
||||||
|
|
||||||
const BATCHES = [
|
const BATCHES = [
|
||||||
{ tag: 'Grok Build CLI · xhigh', label: 'grok-4.6 · 旧盘面', short: 'grok旧', format: 'old' },
|
{ tag: 'Grok Build CLI · xhigh', label: 'grok-4.6 · 旧盘面', short: 'grok旧', format: 'old' },
|
||||||
@@ -44,12 +49,18 @@ const NEW_TAGS = ['pi · Grok 4.6 xhigh', 'pi · Grok 4.6 xhigh · 复测', 'Kim
|
|||||||
|
|
||||||
const runs = readdirSync(RUNS).filter(f => f.endsWith('.json') && f !== 'index.json')
|
const runs = readdirSync(RUNS).filter(f => f.endsWith('.json') && f !== 'index.json')
|
||||||
.map(f => { try { return JSON.parse(readFileSync(resolve(RUNS, f), 'utf8')); } catch { return null; } })
|
.map(f => { try { return JSON.parse(readFileSync(resolve(RUNS, f), 'utf8')); } catch { return null; } })
|
||||||
.filter(Boolean).filter(d => meta[d.level?.id]);
|
.filter(Boolean).filter(d => d.level?.id?.startsWith('proto-d5-'));
|
||||||
|
// Levels retired by a regeneration still have runs worth comparing.
|
||||||
|
let retired = 0;
|
||||||
|
for (const d of runs) if (!meta[d.level.id]) { meta[d.level.id] = describe(d.level, null); retired++; }
|
||||||
|
// Give every level a stable ordinal: current set first, retired ones after.
|
||||||
|
let ord = levels.length;
|
||||||
|
for (const k of Object.keys(meta)) if (meta[k].n == null) meta[k].n = ++ord;
|
||||||
|
|
||||||
const byTag = {};
|
const byTag = {};
|
||||||
for (const b of BATCHES) {
|
for (const b of BATCHES) {
|
||||||
const rows = {};
|
const rows = {};
|
||||||
for (const d of runs) if (d.agent === b.tag) rows[meta[d.level.id].n] = { m: metrics(d), lv: meta[d.level.id], run: d.run };
|
for (const d of runs) if (d.agent === b.tag) rows[d.level.id] = { m: metrics(d), lv: meta[d.level.id], run: d.run, id: d.level.id };
|
||||||
byTag[b.tag] = rows;
|
byTag[b.tag] = rows;
|
||||||
}
|
}
|
||||||
const present = BATCHES.filter(b => Object.keys(byTag[b.tag]).length > 0);
|
const present = BATCHES.filter(b => Object.keys(byTag[b.tag]).length > 0);
|
||||||
@@ -82,7 +93,7 @@ const METRICS = [
|
|||||||
|
|
||||||
function pair(tagA, tagB) {
|
function pair(tagA, tagB) {
|
||||||
const A = byTag[tagA] || {}, B = byTag[tagB] || {};
|
const A = byTag[tagA] || {}, B = byTag[tagB] || {};
|
||||||
const shared = Object.keys(A).filter(k => B[k]).map(Number).sort((a, b) => a - b);
|
const shared = Object.keys(A).filter(k => B[k]).sort((x, y) => meta[x].n - meta[y].n);
|
||||||
return { shared, A, B };
|
return { shared, A, B };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -184,9 +195,9 @@ const THEMES = [
|
|||||||
];
|
];
|
||||||
function notes() {
|
function notes() {
|
||||||
const outp = [];
|
const outp = [];
|
||||||
for (const b of present) for (const [L, r] of Object.entries(byTag[b.tag])) {
|
for (const b of present) for (const r of Object.values(byTag[b.tag])) {
|
||||||
const w = r.m.rating?.worst; if (!w) continue;
|
const w = r.m.rating?.worst; if (!w) continue;
|
||||||
outp.push({ batch: b.short, L: Number(L), fam: r.lv.family, text: String(w) });
|
outp.push({ batch: b.short, L: r.lv.n, fam: r.lv.family, text: String(w) });
|
||||||
}
|
}
|
||||||
return outp.sort((a, b) => a.L - b.L);
|
return outp.sort((a, b) => a.L - b.L);
|
||||||
}
|
}
|
||||||
@@ -203,11 +214,12 @@ const clearRateNote = allCleared
|
|||||||
const liveTags = NEW_TAGS.filter(t => Object.keys(byTag[t] || {}).length);
|
const liveTags = NEW_TAGS.filter(t => Object.keys(byTag[t] || {}).length);
|
||||||
const triple = m => [m.rotations - m.minimumRotations, m.blindRotations, m.wrongWayRotations].join('/');
|
const triple = m => [m.rotations - m.minimumRotations, m.blindRotations, m.wrongWayRotations].join('/');
|
||||||
const crossRows = [];
|
const crossRows = [];
|
||||||
for (let L = 1; L <= levels.length; L++) {
|
const crossIds = [...new Set(liveTags.flatMap(t => Object.keys(byTag[t] || {})))]
|
||||||
const cells = liveTags.map(t => byTag[t][L] ? triple(byTag[t][L].m) : null);
|
.filter(id => liveTags.every(t => byTag[t]?.[id]))
|
||||||
if (cells.some(c => c == null)) continue;
|
.sort((a, b) => meta[a].n - meta[b].n);
|
||||||
const lv = Object.values(meta).find(x => x.n === L);
|
for (const id of crossIds) {
|
||||||
crossRows.push({ L, family: lv.family, cells, agree: new Set(cells).size === 1 });
|
const cells = liveTags.map(t => triple(byTag[t][id].m));
|
||||||
|
crossRows.push({ L: meta[id].n, id, family: meta[id].family, cells, agree: new Set(cells).size === 1 });
|
||||||
}
|
}
|
||||||
const agreeN = crossRows.filter(r => r.agree).length;
|
const agreeN = crossRows.filter(r => r.agree).length;
|
||||||
const crossModel = crossRows.length < 3 ? '<p class="sub">同盘面批次不足，跨模型比较尚未成立。</p>' : `
|
const crossModel = crossRows.length < 3 ? '<p class="sub">同盘面批次不足，跨模型比较尚未成立。</p>' : `
|
||||||
|
|||||||
Reference in New Issue
Block a user