老师最痛恨的"抄作业"，原来是小模型追上大模型的唯一办法

import torch

hard = torch.tensor([1.0, 0.0, 0.0])            # 标准答案：就是猫
soft = torch.tensor([0.90, 0.08, 0.02])         # 老师的判断：像猫，有点像狗

def entropy(p):
    p = p.clamp_min(1e-9)
    return -(p * p.log()).sum().item()

print(f"硬标签 熵 = {entropy(hard):.4f}")        # 0.0000 —— 零信息，只有一个确定结论
print(f"软标签 熵 = {entropy(soft):.4f}")        # 0.3600 —— 携带了"相似度结构"

import torch

classes = ["猫", "狗", "狐狸", "汽车"]
soft = torch.tensor([0.90, 0.07, 0.025, 0.005])

for c, p in zip(classes, soft):
    print(f"{c:>3}: {p:.3f}  {'█' * int(p * 60)}")

  猫: 0.900  ██████████████████████████████████████████████████████
  狗: 0.070  ████
狐狸: 0.025  █
汽车: 0.005  

import torch

# 一张模棱两可的样本
hard = torch.tensor([1.0, 0.0, 0.0])          # 硬标签：必须 100% 确信
soft = torch.tensor([0.40, 0.35, 0.25])       # 老师：我也不太确定

print(f"硬标签要求的置信度: {hard.max():.0%}  -> 逼学生死记")
print(f"软标签要求的置信度: {soft.max():.0%}  -> 允许学生存疑")

import torch, torch.nn.functional as F

logits = torch.tensor([8.0, 2.0, 1.0, -3.0])     # 老师原始输出，非常自信

for T in [1.0, 2.0, 4.0, 8.0]:
    p = F.softmax(logits / T, dim=0)
    print(f"T={T:>3}: {[f'{v:.3f}' for v in p]}")

T=1.0: ['0.997', '0.002', '0.001', '0.000']   <- 暗知识被压没了
T=2.0: ['0.947', '0.047', '0.029', '0.004']
T=4.0: ['0.786', '0.175', '0.136', '0.050']   <- 相对关系清晰可见
T=8.0: ['0.554', '0.261', '0.230', '0.126']

import torch, torch.nn as nn, torch.nn.functional as F

torch.manual_seed(0)
D, C = 20, 5

# 造一个有结构的任务（类别之间有真实的远近关系）
W = torch.randn(D, C)
def make(n):
    X = torch.randn(n, D)
    y = (X @ W).argmax(1)
    return X, y

# ---- 老师：数据充足，学得很好 ----
Xt, yt = make(6000)
teacher = nn.Sequential(nn.Linear(D, 128), nn.ReLU(), nn.Linear(128, C))
opt = torch.optim.Adam(teacher.parameters(), lr=1e-2)
for _ in range(1200):
    loss = F.cross_entropy(teacher(Xt), yt)
    opt.zero_grad(); loss.backward(); opt.step()

# ---- 学生：只有 300 条数据（抄作业的典型处境）----
Xs, ys = make(300)
Xe, ye = make(3000)                                  # 考试集
with torch.no_grad():
    t_logits = teacher(Xs)                           # 老师在这 300 条上的"解题过程"

def student(mode, T=4.0, steps=1500):
    net = nn.Sequential(nn.Linear(D, 32), nn.ReLU(), nn.Linear(32, C))
    opt = torch.optim.Adam(net.parameters(), lr=1e-2)
    for _ in range(steps):
        out = net(Xs)
        if mode == "hard":                           # A：只抄答案
            loss = F.cross_entropy(out, ys)
        else:                                        # B：抄整个分布
            loss = F.kl_div(F.log_softmax(out / T, 1),
                            F.softmax(t_logits / T, 1),
                            reduction="batchmean") * T * T
        opt.zero_grad(); loss.backward(); opt.step()
    with torch.no_grad():
        return (net(Xe).argmax(1) == ye).float().mean().item()

with torch.no_grad():
    print(f"老师       考试准确率: {(teacher(Xe).argmax(1) == ye).float().mean():.1%}")
print(f"A 只抄答案 考试准确率: {student('hard'):.1%}")
print(f"B 抄解题过程 考试准确率: {student('soft'):.1%}")

python distillation_visualization.py