| 1 | <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 720" width="1280" height="720"> |
| 2 | <defs> |
| 3 | <linearGradient id="trackGrad" x1="0%" y1="0%" x2="0%" y2="100%"> |
| 4 | <stop offset="0%" stop-color="#1A365D"/> |
| 5 | <stop offset="100%" stop-color="#3182CE"/> |
| 6 | </linearGradient> |
| 7 | <filter id="cardShadow" x="-5%" y="-15%" width="110%" height="135%"> |
| 8 | <feGaussianBlur in="SourceAlpha" stdDeviation="4"/> |
| 9 | <feOffset dx="0" dy="2"/> |
| 10 | <feFlood flood-color="#1A365D" flood-opacity="0.06"/> |
| 11 | <feComposite in2="SourceAlpha" operator="in"/> |
| 12 | <feMerge><feMergeNode/><feMergeNode in="SourceGraphic"/></feMerge> |
| 13 | </filter> |
| 14 | </defs> |
| 15 | |
| 16 | <g id="bg"> |
| 17 | <rect width="1280" height="720" fill="#FFFFFF"/> |
| 18 | </g> |
| 19 | |
| 20 | <g id="header"> |
| 21 | <rect x="60" y="56" width="6" height="40" fill="#3182CE"/> |
| 22 | <text x="84" y="88" font-family="Georgia, "Microsoft YaHei", serif" font-size="36" font-weight="700" fill="#1A365D">Five takeaways from the variation study</text> |
| 23 | <text x="84" y="118" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="13" font-weight="700" fill="#4A5568" letter-spacing="1.5">TABLE 3 ABLATIONS · WHAT THE ARCHITECTURE NEEDS TO STAY GOOD</text> |
| 24 | </g> |
| 25 | |
| 26 | <g id="track"> |
| 27 | <line x1="160" y1="172" x2="160" y2="660" stroke="url(#trackGrad)" stroke-width="4" stroke-linecap="round"/> |
| 28 | </g> |
| 29 | |
| 30 | <g id="item-1"> |
| 31 | <circle cx="160" cy="200" r="28" fill="#1A365D" fill-opacity="0.12"/> |
| 32 | <circle cx="160" cy="200" r="20" fill="#1A365D"/> |
| 33 | <text x="160" y="206" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">01</text> |
| 34 | <line x1="188" y1="200" x2="220" y2="200" stroke="#1A365D" stroke-width="2" stroke-dasharray="4,3"/> |
| 35 | <rect x="220" y="170" width="1000" height="64" rx="10" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 36 | <rect x="220" y="170" width="6" height="64" rx="3" fill="#1A365D"/> |
| 37 | <text x="246" y="196" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Head count has a sweet spot</text> |
| 38 | <text x="246" y="222" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Single head loses <tspan font-family="Consolas, monospace" font-weight="700" fill="#C53030">−0.9 BLEU</tspan> vs h = 8; too many heads (h = 32) also degrade. Eight is empirically right.</text> |
| 39 | </g> |
| 40 | |
| 41 | <g id="item-2"> |
| 42 | <circle cx="160" cy="296" r="28" fill="#1A365D" fill-opacity="0.12"/> |
| 43 | <circle cx="160" cy="296" r="20" fill="#1A365D"/> |
| 44 | <text x="160" y="302" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">02</text> |
| 45 | <line x1="188" y1="296" x2="220" y2="296" stroke="#1A365D" stroke-width="2" stroke-dasharray="4,3"/> |
| 46 | <rect x="220" y="266" width="1000" height="64" rx="10" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 47 | <rect x="220" y="266" width="6" height="64" rx="3" fill="#1A365D"/> |
| 48 | <text x="246" y="292" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Key dim matters more than value dim</text> |
| 49 | <text x="246" y="318" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Reducing <tspan font-family="Consolas, monospace" font-weight="700" fill="#3182CE">d_k</tspan> hurts quality more than reducing <tspan font-family="Consolas, monospace" font-weight="700" fill="#3182CE">d_v</tspan> — determining compatibility is the harder learning problem.</text> |
| 50 | </g> |
| 51 | |
| 52 | <g id="item-3"> |
| 53 | <circle cx="160" cy="392" r="28" fill="#3182CE" fill-opacity="0.12"/> |
| 54 | <circle cx="160" cy="392" r="20" fill="#3182CE"/> |
| 55 | <text x="160" y="398" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">03</text> |
| 56 | <line x1="188" y1="392" x2="220" y2="392" stroke="#3182CE" stroke-width="2" stroke-dasharray="4,3"/> |
| 57 | <rect x="220" y="362" width="1000" height="64" rx="10" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 58 | <rect x="220" y="362" width="6" height="64" rx="3" fill="#3182CE"/> |
| 59 | <text x="246" y="388" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Bigger is better</text> |
| 60 | <text x="246" y="414" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Larger <tspan font-family="Consolas, monospace" font-weight="700" fill="#3182CE">d_model</tspan> and deeper stacks consistently improve BLEU — the architecture scales monotonically with capacity.</text> |
| 61 | </g> |
| 62 | |
| 63 | <g id="item-4"> |
| 64 | <circle cx="160" cy="488" r="28" fill="#3182CE" fill-opacity="0.12"/> |
| 65 | <circle cx="160" cy="488" r="20" fill="#3182CE"/> |
| 66 | <text x="160" y="494" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">04</text> |
| 67 | <line x1="188" y1="488" x2="220" y2="488" stroke="#3182CE" stroke-width="2" stroke-dasharray="4,3"/> |
| 68 | <rect x="220" y="458" width="1000" height="64" rx="10" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 69 | <rect x="220" y="458" width="6" height="64" rx="3" fill="#3182CE"/> |
| 70 | <text x="246" y="484" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Dropout is essential</text> |
| 71 | <text x="246" y="510" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Removing dropout loses <tspan font-family="Consolas, monospace" font-weight="700" fill="#C53030">~0.5–1.0 BLEU</tspan>. Even p = 0.1 measurably regularizes — it isn't decorative.</text> |
| 72 | </g> |
| 73 | |
| 74 | <g id="item-5"> |
| 75 | <circle cx="160" cy="584" r="28" fill="#63B3ED" fill-opacity="0.18"/> |
| 76 | <circle cx="160" cy="584" r="20" fill="#3182CE"/> |
| 77 | <text x="160" y="590" text-anchor="middle" font-family="Georgia, "Microsoft YaHei", serif" font-size="15" font-weight="700" fill="#FFFFFF">05</text> |
| 78 | <line x1="188" y1="584" x2="220" y2="584" stroke="#3182CE" stroke-width="2" stroke-dasharray="4,3"/> |
| 79 | <rect x="220" y="554" width="1000" height="64" rx="10" fill="#FFFFFF" stroke="#E2E8F0" stroke-width="1.5" filter="url(#cardShadow)"/> |
| 80 | <rect x="220" y="554" width="6" height="64" rx="3" fill="#63B3ED"/> |
| 81 | <text x="246" y="580" font-family="Georgia, "Microsoft YaHei", serif" font-size="18" font-weight="700" fill="#1A365D">Label smoothing helps BLEU (and hurts perplexity)</text> |
| 82 | <text x="246" y="606" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="14" fill="#1A202C">Slightly worse perplexity, better BLEU. <tspan font-style="italic" fill="#4A5568">Optimize for the metric you ship on.</tspan></text> |
| 83 | </g> |
| 84 | |
| 85 | <g id="footer"> |
| 86 | <text x="60" y="694" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">Source: Vaswani et al. 2017, Table 3 (§6.2 Model Variations)</text> |
| 87 | <text x="1220" y="694" text-anchor="end" font-family="Arial, "Microsoft YaHei", sans-serif" font-size="11" fill="#A0AEC0">15</text> |
| 88 | </g> |
| 89 | </svg> |
| 90 |