Appearance
nn.py
复制以下内容并保存为 nn.py,使用方法见配套代码说明。
python
"""一个纯 Python 实现的多层感知机(MLP),不使用 NumPy 或任何第三方库。
本文件配套第 3 章《神经网络与反向传播》。设计目标有三个:
1. **一切可见**:矩阵就是 list of list,矩阵乘法就是三层 for 循环,
每个数学符号都能在代码里找到对应的一行。
2. **梯度核验**:用小网络比较解析梯度与中心有限差分,结合绝对误差、
相对误差和扰动步长判断一致性。
3. **教学完整**:包含前向传播、反向传播、mini-batch 梯度下降、
动量、损失曲线记录、参数扁平化(用于梯度检查)。
记号约定(与教程正文一致)
----------------------------
网络共 L 层权重。第 0 层是输入,第 L 层是输出。
a^0 = x (输入)
z^l = W^l · a^(l-1) + b^l (第 l 层的加权和)
a^l = phi(z^l) (第 l 层的激活)
J = loss(a^L, y) (损失)
其中 W^l 的形状是 (n_l, n_{l-1}),b^l 的形状是 (n_l,)。
代码里 self.W[l]、self.b[l] 的下标比数学记号里的 l 小 1,
因为 self.W[0] 对应数学上的 W^1。
"""
from __future__ import annotations
import math
import random
from typing import Callable, Dict, List, Optional, Sequence, Tuple
Vector = List[float]
Matrix = List[List[float]]
# ---------------------------------------------------------------------------
# 1. 矩阵工具:全部手写,方便读者逐行对照
# ---------------------------------------------------------------------------
def zeros(n: int, m: int) -> Matrix:
"""返回 n 行 m 列的全零矩阵。"""
return [[0.0] * m for _ in range(n)]
def matmul(A: Matrix, B: Matrix) -> Matrix:
"""矩阵乘法 A @ B。A 是 (n, m),B 是 (m, p),返回 (n, p)。"""
n = len(A)
m = len(B)
p = len(B[0]) if B else 0
out = zeros(n, p)
for i in range(n):
Ai = A[i]
outi = out[i]
for k in range(m):
a = Ai[k]
if a == 0.0:
continue
Bk = B[k]
for j in range(p):
outi[j] += a * Bk[j]
return out
def matvec(A: Matrix, x: Vector) -> Vector:
"""矩阵乘以列向量 A @ x。A 是 (n, m),x 是 (m,),返回 (n,)。"""
return [sum(A[i][j] * x[j] for j in range(len(x))) for i in range(len(A))]
def transpose(A: Matrix) -> Matrix:
"""转置。A 是 (n, m),返回 (m, n)。"""
return [list(col) for col in zip(*A)]
def outer(u: Vector, v: Vector) -> Matrix:
"""外积 u v^T。u 是 (n,),v 是 (m,),返回 (n, m)。"""
return [[ui * vj for vj in v] for ui in u]
def hadamard(u: Vector, v: Vector) -> Vector:
"""逐元素乘积(Hadamard 积)。"""
return [ui * vi for ui, vi in zip(u, v)]
def vec_add(u: Vector, v: Vector) -> Vector:
return [ui + vi for ui, vi in zip(u, v)]
def vec_scale(u: Vector, s: float) -> Vector:
return [ui * s for ui in u]
# ---------------------------------------------------------------------------
# 2. 激活函数:每个都给出值和导数
# ---------------------------------------------------------------------------
def sigmoid(z: Vector) -> Vector:
out = []
for zi in z:
if zi >= 0: # 数值稳定的写法,避免 exp 溢出
out.append(1.0 / (1.0 + math.exp(-zi)))
else:
e = math.exp(zi)
out.append(e / (1.0 + e))
return out
def tanh(z: Vector) -> Vector:
return [math.tanh(zi) for zi in z]
def relu(z: Vector) -> Vector:
return [zi if zi > 0.0 else 0.0 for zi in z]
def identity(z: Vector) -> Vector:
return list(z)
# 导数都用「激活值 a」或「加权和 z」计算,避免重复 exp
ACTIVATIONS: Dict[str, Callable[[Vector], Vector]] = {
"sigmoid": sigmoid,
"tanh": tanh,
"relu": relu,
"linear": identity,
}
def activation_prime(name: str, z: Vector, a: Vector) -> Vector:
"""激活函数对 z 的导数。a = phi(z) 已经算好,直接复用。"""
if name == "sigmoid":
return [ai * (1.0 - ai) for ai in a] # sigma' = sigma(1-sigma)
if name == "tanh":
return [1.0 - ai * ai for ai in a] # tanh' = 1 - tanh^2
if name == "relu":
return [1.0 if zi > 0.0 else 0.0 for zi in z] # z=0 不可微,代码取次梯度 0
return [1.0] * len(z) # 恒等:导数为 1
# ---------------------------------------------------------------------------
# 3. 多层感知机
# ---------------------------------------------------------------------------
class MLP:
"""全连接前馈网络。
参数
----
sizes : 每层的神经元个数,例如 [2, 4, 1] 表示 2 输入、1 个 4 神经元隐层、1 输出。
hidden_activation : 隐层激活函数,'tanh' / 'relu' / 'sigmoid'。
output_activation : 输出层激活函数,'linear'(回归)或 'sigmoid'(二分类)。
loss : 'mse'(均方误差)或 'bce'(二元交叉熵,需配合 sigmoid 输出)。
seed : 随机种子,保证结果可复现。
"""
def __init__(
self,
sizes: Sequence[int],
hidden_activation: str = "tanh",
output_activation: str = "linear",
loss: str = "mse",
seed: int = 0,
) -> None:
if len(sizes) < 2:
raise ValueError("至少需要输入层和输出层两层")
self.sizes = list(sizes)
self.hidden_activation = hidden_activation
self.output_activation = output_activation
self.loss_name = loss
if hidden_activation not in ACTIVATIONS:
raise ValueError(f"未知激活函数: {hidden_activation}")
if output_activation not in ACTIVATIONS:
raise ValueError(f"未知激活函数: {output_activation}")
if loss not in ("mse", "bce"):
raise ValueError(f"未知损失函数: {loss}")
if loss == "bce" and output_activation != "sigmoid":
raise ValueError("bce 损失需要 sigmoid 输出,以联合计算 logits 损失与梯度")
rng = random.Random(seed)
self.W: List[Matrix] = []
self.b: List[Vector] = []
n_layers = len(sizes) - 1
for l in range(n_layers):
fan_in, fan_out = sizes[l], sizes[l + 1]
is_output = (l == n_layers - 1)
# ReLU 隐藏层采用 He 正态初始化,方差为 2/fan_in。
# 其余层采用 LeCun 正态尺度,方差为 1/fan_in。
# 这里抽取未截断高斯;Glorot/Xavier 的基准方差为 2/(fan_in+fan_out)。
if (not is_output) and hidden_activation == "relu":
scale = math.sqrt(2.0 / fan_in)
else:
scale = math.sqrt(1.0 / fan_in)
self.W.append(
[[rng.gauss(0.0, scale) for _ in range(fan_in)] for _ in range(fan_out)]
)
self.b.append([0.0] * fan_out)
# ---------------- 前向传播 ----------------
def forward(self, x: Vector) -> Tuple[List[Vector], List[Vector]]:
"""返回 (zs, activations)。zs[l] 对应数学层 l+1 的加权和,activations[0] = x。"""
a = list(x)
zs: List[Vector] = []
activations: List[Vector] = [a]
n_layers = len(self.W)
for l in range(n_layers):
z = matvec(self.W[l], a)
b = self.b[l]
z = [z[j] + b[j] for j in range(len(z))]
is_output = (l == n_layers - 1)
name = self.output_activation if is_output else self.hidden_activation
a = ACTIVATIONS[name](z)
zs.append(z)
activations.append(a)
return zs, activations
def predict(self, x: Vector) -> Vector:
"""只返回输出层的激活值。"""
_, activations = self.forward(x)
return activations[-1]
# ---------------- 损失 ----------------
def loss_and_delta(self, z: Vector, a: Vector, y: Vector) -> Tuple[float, Vector]:
"""返回单样本损失 J 及输出层误差 dJ/dz。
BCE 直接用 logits z 计算稳定损失,联合导数为 sigmoid(z)-y;
MSE 对输出分量求和后乘 1/2,再乘输出激活的局部导数。
"""
if self.loss_name == "bce":
J = sum(max(zi, 0.0) - zi * yi + math.log1p(math.exp(-abs(zi)))
for zi, yi in zip(z, y))
return J, [ai - yi for ai, yi in zip(a, y)]
J = 0.5 * sum((ai - yi) ** 2 for ai, yi in zip(a, y))
da = [ai - yi for ai, yi in zip(a, y)]
return J, hadamard(da, activation_prime(self.output_activation, z, a))
def loss_on(self, x: Vector, y: Vector) -> float:
"""单个样本的损失,供数值梯度检查使用。"""
zs, activations = self.forward(x)
J, _ = self.loss_and_delta(zs[-1], activations[-1], y)
return J
# ---------------- 反向传播 ----------------
def backward(self, x: Vector, y: Vector) -> Tuple[float, List[Matrix], List[Vector]]:
"""对一个样本做前向 + 反向,返回 (J, dW, db)。
核心公式(l 从 1 到 L,代码下标从 0 到 L-1):
delta^L = dJ/da^L ⊙ phi'(z^L) (逐元素输出激活;BCE 联合化简为 a-y)
delta^l = (W^(l+1))^T · delta^(l+1) ⊙ phi'(z^l)(误差反传)
dJ/dW^l = delta^l · (a^(l-1))^T (权重梯度)
dJ/db^l = delta^l (偏置梯度)
"""
zs, activations = self.forward(x)
n_layers = len(self.W)
J, output_delta = self.loss_and_delta(zs[-1], activations[-1], y)
deltas: List[Vector] = [None] * n_layers # type: ignore[list-item]
# (1) 输出层误差:BCE 分支已联合求导,不再乘 sigmoid 导数
deltas[-1] = output_delta
# (2) 误差逐层回传
for l in range(n_layers - 2, -1, -1):
W_next = self.W[l + 1] # 形状 (sizes[l+2], sizes[l+1])
d_next = deltas[l + 1] # 长度 sizes[l+2]
# 把上游误差按权重回传:sum_k W_next[k][j] * delta_next[k]
back = [
sum(W_next[k][j] * d_next[k] for k in range(len(d_next)))
for j in range(len(self.W[l]))
]
prime = activation_prime(
self.hidden_activation, zs[l], activations[l + 1]
)
deltas[l] = hadamard(back, prime)
# (3) 权重与偏置梯度
dW = [outer(deltas[l], activations[l]) for l in range(n_layers)]
db = [list(deltas[l]) for l in range(n_layers)]
return J, dW, db
# ---------------- 训练 ----------------
def train(
self,
X: Sequence[Vector],
Y: Sequence[Vector],
epochs: int = 2000,
lr: float = 0.5,
batch_size: Optional[int] = None,
momentum: float = 0.0,
shuffle: bool = True,
seed: int = 0,
verbose: bool = False,
log_every: int = 200,
) -> List[float]:
"""mini-batch 梯度下降。返回每个 epoch 的平均损失。"""
n = len(X)
batch_size = batch_size or n
rng = random.Random(seed)
vW = [zeros(len(self.W[l]), len(self.W[l][0])) for l in range(len(self.W))]
vb = [ [0.0] * len(self.b[l]) for l in range(len(self.b)) ]
history: List[float] = []
for epoch in range(1, epochs + 1):
order = list(range(n))
if shuffle:
rng.shuffle(order)
epoch_loss = 0.0
for start in range(0, n, batch_size):
idxs = order[start:start + batch_size]
m = len(idxs)
gW = [zeros(len(self.W[l]), len(self.W[l][0])) for l in range(len(self.W))]
gb = [ [0.0] * len(self.b[l]) for l in range(len(self.b)) ]
for i in idxs:
J, dW, db = self.backward(X[i], Y[i])
epoch_loss += J
for l in range(len(self.W)):
gl, dl = gW[l], dW[l]
for r in range(len(gl)):
gr, dr = gl[r], dl[r]
for c in range(len(gr)):
gr[c] += dr[c]
gbl, dbl = gb[l], db[l]
for r in range(len(gbl)):
gbl[r] += dbl[r]
# 平均 + 更新(带动量)
for l in range(len(self.W)):
gl, vl, Wl = gW[l], vW[l], self.W[l]
for r in range(len(gl)):
for c in range(len(gl[r])):
g = gl[r][c] / m
vl[r][c] = momentum * vl[r][c] - lr * g
Wl[r][c] += vl[r][c]
gbl, vbl, bl = gb[l], vb[l], self.b[l]
for r in range(len(gbl)):
g = gbl[r] / m
vbl[r] = momentum * vbl[r] - lr * g
bl[r] += vbl[r]
history.append(epoch_loss / n)
if verbose and (epoch % log_every == 0 or epoch == 1):
print(f"epoch {epoch:5d} loss = {history[-1]:.6e}")
return history
# ---------------- 参数扁平化:梯度检查用 ----------------
def get_params(self) -> Vector:
flat: Vector = []
for W in self.W:
for row in W:
flat.extend(row)
for b in self.b:
flat.extend(b)
return flat
def set_params(self, flat: Vector) -> None:
k = 0
for W in self.W:
for r in range(len(W)):
for c in range(len(W[r])):
W[r][c] = flat[k]
k += 1
for b in self.b:
for r in range(len(b)):
b[r] = flat[k]
k += 1
def n_params(self) -> int:
return len(self.get_params())
# ---------------------------------------------------------------------------
# 4. 梯度检查:解析梯度 vs 有限差分
# ---------------------------------------------------------------------------
def numeric_gradient(
net: MLP, X: Sequence[Vector], Y: Sequence[Vector], eps: float = 1e-6
) -> Vector:
"""用中心差分估计 d(平均损失)/d(theta)。"""
flat = net.get_params()
grads: Vector = []
m = len(X)
for i in range(len(flat)):
original = flat[i]
flat[i] = original + eps
net.set_params(flat)
plus = sum(net.loss_on(X[k], Y[k]) for k in range(m)) / m
flat[i] = original - eps
net.set_params(flat)
minus = sum(net.loss_on(X[k], Y[k]) for k in range(m)) / m
flat[i] = original
grads.append((plus - minus) / (2.0 * eps))
net.set_params(flat)
return grads
def analytic_gradient(net: MLP, X: Sequence[Vector], Y: Sequence[Vector]) -> Vector:
"""把 backward 的解析梯度按与 get_params 相同的顺序展平。"""
m = len(X)
gW = [zeros(len(net.W[l]), len(net.W[l][0])) for l in range(len(net.W))]
gb = [[0.0] * len(net.b[l]) for l in range(len(net.b))]
for k in range(m):
_, dW, db = net.backward(X[k], Y[k])
for l in range(len(net.W)):
for r in range(len(dW[l])):
for c in range(len(dW[l][r])):
gW[l][r][c] += dW[l][r][c]
for r in range(len(db[l])):
gb[l][r] += db[l][r]
flat: Vector = []
for l in range(len(gW)):
for row in gW[l]:
flat.extend(v / m for v in row)
for l in range(len(gb)):
flat.extend(v / m for v in gb[l])
return flat
def check_gradient(
net: MLP,
X: Sequence[Vector],
Y: Sequence[Vector],
eps: float = 1e-6,
) -> float:
"""返回解析梯度与数值梯度的最大相对误差。"""
ana = analytic_gradient(net, X, Y)
num = numeric_gradient(net, X, Y, eps=eps)
worst = 0.0
for a, n in zip(ana, num):
denom = max(1e-8, abs(a) + abs(n))
worst = max(worst, abs(a - n) / denom)
return worst