为什么大模型能堆到几十层还训得动？答案藏在一个最不起眼的动作里

import torch

scores = torch.tensor([88.0, 92.0, 79.0, 95.0, 61.0])   # 一组原始数值
z = (scores - scores.mean()) / scores.std(unbiased=False)  # 减均值，除标准差
print("标准化后:", z.round(decimals=2))
print("新均值≈", z.mean().round(decimals=2), " 新标准差≈", z.std(unbiased=False).round(decimals=2))
# 新均值≈0，新标准差≈1 —— 无论原来什么量级，都被拉回同一个标准状态

import torch

subject_a = torch.tensor([110.0, 95.0, 120.0, 88.0])   # A 项，量级偏大
subject_b = torch.tensor([70.0,  55.0, 82.0,  60.0])   # B 项，量级偏小

def zscore(x):
    return (x - x.mean()) / x.std(unbiased=False)

print("A 项标准化:", zscore(subject_a).round(decimals=2))
print("B 项标准化:", zscore(subject_b).round(decimals=2))
# 两项被拉到同一把尺子：谁在自己那组里更靠前，一眼就能比

import torch

x = torch.randn(1000) * 20 + 100     # 均值≈100、标准差≈20 的一堆数
print("原始:  均值≈", x.mean().round(), " 标准差≈", x.std(unbiased=False).round())

x_centered = x - x.mean()            # 动作一：整体平移，把中心搬到 0
x_norm = x_centered / x.std(unbiased=False)   # 动作二：整体缩放，把波动压到 1
print("标准化: 均值≈", x_norm.mean().round(), " 标准差≈", x_norm.std(unbiased=False).round())
# 均值≈0，标准差≈1：钟形曲线还是那个钟形，只是被搬到原点、宽度收成了 1

import torch

torch.manual_seed(0)
x = torch.randn(1, 64)
for layer in range(1, 51):
    W = torch.randn(64, 64) * 0.6      # 一个普通的线性层
    x = torch.relu(x @ W)              # 乘矩阵 + 激活，模拟往深处传
    if layer % 10 == 0:
        print(f"第 {layer:2d} 层后：数值绝对值均值 ≈ {x.abs().mean():.4f}")
# 你会看到数值一路飘走（膨胀或塌缩），越深越离谱

import torch
import torch.nn.functional as F

torch.manual_seed(0)
x = torch.randn(1, 64)
for layer in range(1, 51):
    W = torch.randn(64, 64) * 0.6
    x = torch.relu(x @ W)
    x = (x - x.mean(-1, keepdim=True)) / (x.std(-1, keepdim=True) + 1e-5)  # 每层归一化
    if layer % 10 == 0:
        print(f"第 {layer:2d} 层后：数值绝对值均值 ≈ {x.abs().mean():.4f}")
# 加了归一化，数值稳稳待在一个区间，50 层也不飘

import torch
import torch.nn as nn

torch.manual_seed(0)
x = torch.randn(2, 4)                 # 2 个词，每个词是一个 4 维向量

# —— 手动版：就是"减均值、除标准差" ——
mean = x.mean(dim=-1, keepdim=True)
std  = x.var(dim=-1, keepdim=True, unbiased=False).sqrt()
manual = (x - mean) / (std + 1e-5)

# —— PyTorch 官方版 ——
ln = nn.LayerNorm(4, elementwise_affine=False)   # 先关掉 γ/β，只做纯标准化
official = ln(x)

print("手动 vs 官方 完全一致:", torch.allclose(manual, official, atol=1e-5))  # True

import torch
import torch.nn as nn

torch.manual_seed(0)
x = torch.randn(2, 4)

ln = nn.LayerNorm(4)          # 默认带 γ、β
print("初始 γ:", ln.weight.data)   # 全 1
print("初始 β:", ln.bias.data)     # 全 0
# 初始时 γ=1、β=0，等于"只做纯标准化"；训练中它们会被学成模型需要的值

# 手动复现完整版：标准化后再 * γ + β
mean = x.mean(-1, keepdim=True)
std  = x.var(-1, keepdim=True, unbiased=False).sqrt()
manual = (x - mean) / (std + 1e-5) * ln.weight + ln.bias
print("和官方完全一致:", torch.allclose(manual, ln(x), atol=1e-5))   # True

import torch
import torch.nn as nn

torch.manual_seed(0)
d = 16
ln = nn.LayerNorm(d)
linear = nn.Linear(d, d)

x = torch.randn(4, d) * 5          # 故意给一个量级偏大的输入
print("输入量级(绝对值均值):", x.abs().mean().round(decimals=2))

# 无归一化：直接过子层
no_norm = x + torch.relu(linear(x))
print("无归一化 输出量级:", no_norm.abs().mean().round(decimals=2))

# 有归一化（pre-norm，现代 Transformer 的标准做法）：先归一化再进子层
pre_norm = x + torch.relu(linear(ln(x)))
print("有归一化 输出量级:", pre_norm.abs().mean().round(decimals=2))
# 有归一化的那条，子层始终面对"标准状态"的输入，量级被牢牢摁住

python layernorm_visualization.py