图片详情
—
Skip to content

nn.py ​

返回章节 · 配套代码说明

复制以下内容并保存为 nn.py,使用方法见配套代码说明。

python
"""一个纯 Python 实现的多层感知机(MLP),不使用 NumPy 或任何第三方库。

本文件配套第 3 章《神经网络与反向传播》。设计目标有三个:

1. **一切可见**:矩阵就是 list of list,矩阵乘法就是三层 for 循环,
   每个数学符号都能在代码里找到对应的一行。
2. **梯度核验**:用小网络比较解析梯度与中心有限差分,结合绝对误差、
   相对误差和扰动步长判断一致性。
3. **教学完整**:包含前向传播、反向传播、mini-batch 梯度下降、
   动量、损失曲线记录、参数扁平化(用于梯度检查)。

记号约定(与教程正文一致)
----------------------------
网络共 L 层权重。第 0 层是输入,第 L 层是输出。

    a^0       = x                                  (输入)
    z^l       = W^l · a^(l-1) + b^l                (第 l 层的加权和)
    a^l       = phi(z^l)                           (第 l 层的激活)
    J         = loss(a^L, y)                       (损失)

其中 W^l 的形状是 (n_l, n_{l-1}),b^l 的形状是 (n_l,)。
代码里 self.W[l]、self.b[l] 的下标比数学记号里的 l 小 1,
因为 self.W[0] 对应数学上的 W^1。
"""

from __future__ import annotations

import math
import random
from typing import Callable, Dict, List, Optional, Sequence, Tuple

Vector = List[float]
Matrix = List[List[float]]


# ---------------------------------------------------------------------------
# 1. 矩阵工具:全部手写,方便读者逐行对照
# ---------------------------------------------------------------------------

def zeros(n: int, m: int) -> Matrix:
    """返回 n 行 m 列的全零矩阵。"""
    return [[0.0] * m for _ in range(n)]


def matmul(A: Matrix, B: Matrix) -> Matrix:
    """矩阵乘法 A @ B。A 是 (n, m),B 是 (m, p),返回 (n, p)。"""
    n = len(A)
    m = len(B)
    p = len(B[0]) if B else 0
    out = zeros(n, p)
    for i in range(n):
        Ai = A[i]
        outi = out[i]
        for k in range(m):
            a = Ai[k]
            if a == 0.0:
                continue
            Bk = B[k]
            for j in range(p):
                outi[j] += a * Bk[j]
    return out


def matvec(A: Matrix, x: Vector) -> Vector:
    """矩阵乘以列向量 A @ x。A 是 (n, m),x 是 (m,),返回 (n,)。"""
    return [sum(A[i][j] * x[j] for j in range(len(x))) for i in range(len(A))]


def transpose(A: Matrix) -> Matrix:
    """转置。A 是 (n, m),返回 (m, n)。"""
    return [list(col) for col in zip(*A)]


def outer(u: Vector, v: Vector) -> Matrix:
    """外积 u v^T。u 是 (n,),v 是 (m,),返回 (n, m)。"""
    return [[ui * vj for vj in v] for ui in u]


def hadamard(u: Vector, v: Vector) -> Vector:
    """逐元素乘积(Hadamard 积)。"""
    return [ui * vi for ui, vi in zip(u, v)]


def vec_add(u: Vector, v: Vector) -> Vector:
    return [ui + vi for ui, vi in zip(u, v)]


def vec_scale(u: Vector, s: float) -> Vector:
    return [ui * s for ui in u]


# ---------------------------------------------------------------------------
# 2. 激活函数:每个都给出值和导数
# ---------------------------------------------------------------------------

def sigmoid(z: Vector) -> Vector:
    out = []
    for zi in z:
        if zi >= 0:                      # 数值稳定的写法,避免 exp 溢出
            out.append(1.0 / (1.0 + math.exp(-zi)))
        else:
            e = math.exp(zi)
            out.append(e / (1.0 + e))
    return out


def tanh(z: Vector) -> Vector:
    return [math.tanh(zi) for zi in z]


def relu(z: Vector) -> Vector:
    return [zi if zi > 0.0 else 0.0 for zi in z]


def identity(z: Vector) -> Vector:
    return list(z)


# 导数都用「激活值 a」或「加权和 z」计算,避免重复 exp
ACTIVATIONS: Dict[str, Callable[[Vector], Vector]] = {
    "sigmoid": sigmoid,
    "tanh": tanh,
    "relu": relu,
    "linear": identity,
}


def activation_prime(name: str, z: Vector, a: Vector) -> Vector:
    """激活函数对 z 的导数。a = phi(z) 已经算好,直接复用。"""
    if name == "sigmoid":
        return [ai * (1.0 - ai) for ai in a]          # sigma' = sigma(1-sigma)
    if name == "tanh":
        return [1.0 - ai * ai for ai in a]            # tanh' = 1 - tanh^2
    if name == "relu":
        return [1.0 if zi > 0.0 else 0.0 for zi in z]  # z=0 不可微,代码取次梯度 0
    return [1.0] * len(z)                              # 恒等:导数为 1


# ---------------------------------------------------------------------------
# 3. 多层感知机
# ---------------------------------------------------------------------------

class MLP:
    """全连接前馈网络。

    参数
    ----
    sizes : 每层的神经元个数,例如 [2, 4, 1] 表示 2 输入、1 个 4 神经元隐层、1 输出。
    hidden_activation : 隐层激活函数,'tanh' / 'relu' / 'sigmoid'。
    output_activation : 输出层激活函数,'linear'(回归)或 'sigmoid'(二分类)。
    loss : 'mse'(均方误差)或 'bce'(二元交叉熵,需配合 sigmoid 输出)。
    seed : 随机种子,保证结果可复现。
    """

    def __init__(
        self,
        sizes: Sequence[int],
        hidden_activation: str = "tanh",
        output_activation: str = "linear",
        loss: str = "mse",
        seed: int = 0,
    ) -> None:
        if len(sizes) < 2:
            raise ValueError("至少需要输入层和输出层两层")
        self.sizes = list(sizes)
        self.hidden_activation = hidden_activation
        self.output_activation = output_activation
        self.loss_name = loss
        if hidden_activation not in ACTIVATIONS:
            raise ValueError(f"未知激活函数: {hidden_activation}")
        if output_activation not in ACTIVATIONS:
            raise ValueError(f"未知激活函数: {output_activation}")
        if loss not in ("mse", "bce"):
            raise ValueError(f"未知损失函数: {loss}")

        if loss == "bce" and output_activation != "sigmoid":
            raise ValueError("bce 损失需要 sigmoid 输出,以联合计算 logits 损失与梯度")

        rng = random.Random(seed)
        self.W: List[Matrix] = []
        self.b: List[Vector] = []
        n_layers = len(sizes) - 1
        for l in range(n_layers):
            fan_in, fan_out = sizes[l], sizes[l + 1]
            is_output = (l == n_layers - 1)
            # ReLU 隐藏层采用 He 正态初始化,方差为 2/fan_in。
            # 其余层采用 LeCun 正态尺度,方差为 1/fan_in。
            # 这里抽取未截断高斯;Glorot/Xavier 的基准方差为 2/(fan_in+fan_out)。
            if (not is_output) and hidden_activation == "relu":
                scale = math.sqrt(2.0 / fan_in)
            else:
                scale = math.sqrt(1.0 / fan_in)
            self.W.append(
                [[rng.gauss(0.0, scale) for _ in range(fan_in)] for _ in range(fan_out)]
            )
            self.b.append([0.0] * fan_out)

    # ---------------- 前向传播 ----------------
    def forward(self, x: Vector) -> Tuple[List[Vector], List[Vector]]:
        """返回 (zs, activations)。zs[l] 对应数学层 l+1 的加权和,activations[0] = x。"""
        a = list(x)
        zs: List[Vector] = []
        activations: List[Vector] = [a]
        n_layers = len(self.W)
        for l in range(n_layers):
            z = matvec(self.W[l], a)
            b = self.b[l]
            z = [z[j] + b[j] for j in range(len(z))]
            is_output = (l == n_layers - 1)
            name = self.output_activation if is_output else self.hidden_activation
            a = ACTIVATIONS[name](z)
            zs.append(z)
            activations.append(a)
        return zs, activations

    def predict(self, x: Vector) -> Vector:
        """只返回输出层的激活值。"""
        _, activations = self.forward(x)
        return activations[-1]

    # ---------------- 损失 ----------------
    def loss_and_delta(self, z: Vector, a: Vector, y: Vector) -> Tuple[float, Vector]:
        """返回单样本损失 J 及输出层误差 dJ/dz。

        BCE 直接用 logits z 计算稳定损失,联合导数为 sigmoid(z)-y;
        MSE 对输出分量求和后乘 1/2,再乘输出激活的局部导数。
        """
        if self.loss_name == "bce":
            J = sum(max(zi, 0.0) - zi * yi + math.log1p(math.exp(-abs(zi)))
                    for zi, yi in zip(z, y))
            return J, [ai - yi for ai, yi in zip(a, y)]
        J = 0.5 * sum((ai - yi) ** 2 for ai, yi in zip(a, y))
        da = [ai - yi for ai, yi in zip(a, y)]
        return J, hadamard(da, activation_prime(self.output_activation, z, a))

    def loss_on(self, x: Vector, y: Vector) -> float:
        """单个样本的损失,供数值梯度检查使用。"""
        zs, activations = self.forward(x)
        J, _ = self.loss_and_delta(zs[-1], activations[-1], y)
        return J

    # ---------------- 反向传播 ----------------
    def backward(self, x: Vector, y: Vector) -> Tuple[float, List[Matrix], List[Vector]]:
        """对一个样本做前向 + 反向,返回 (J, dW, db)。

        核心公式(l 从 1 到 L,代码下标从 0 到 L-1):

            delta^L   = dJ/da^L  ⊙  phi'(z^L)                (逐元素输出激活;BCE 联合化简为 a-y)
            delta^l   = (W^(l+1))^T · delta^(l+1) ⊙ phi'(z^l)(误差反传)
            dJ/dW^l   = delta^l · (a^(l-1))^T                 (权重梯度)
            dJ/db^l   = delta^l                               (偏置梯度)
        """
        zs, activations = self.forward(x)
        n_layers = len(self.W)

        J, output_delta = self.loss_and_delta(zs[-1], activations[-1], y)

        deltas: List[Vector] = [None] * n_layers  # type: ignore[list-item]

        # (1) 输出层误差:BCE 分支已联合求导,不再乘 sigmoid 导数
        deltas[-1] = output_delta

        # (2) 误差逐层回传
        for l in range(n_layers - 2, -1, -1):
            W_next = self.W[l + 1]          # 形状 (sizes[l+2], sizes[l+1])
            d_next = deltas[l + 1]          # 长度 sizes[l+2]
            # 把上游误差按权重回传:sum_k W_next[k][j] * delta_next[k]
            back = [
                sum(W_next[k][j] * d_next[k] for k in range(len(d_next)))
                for j in range(len(self.W[l]))
            ]
            prime = activation_prime(
                self.hidden_activation, zs[l], activations[l + 1]
            )
            deltas[l] = hadamard(back, prime)

        # (3) 权重与偏置梯度
        dW = [outer(deltas[l], activations[l]) for l in range(n_layers)]
        db = [list(deltas[l]) for l in range(n_layers)]
        return J, dW, db

    # ---------------- 训练 ----------------
    def train(
        self,
        X: Sequence[Vector],
        Y: Sequence[Vector],
        epochs: int = 2000,
        lr: float = 0.5,
        batch_size: Optional[int] = None,
        momentum: float = 0.0,
        shuffle: bool = True,
        seed: int = 0,
        verbose: bool = False,
        log_every: int = 200,
    ) -> List[float]:
        """mini-batch 梯度下降。返回每个 epoch 的平均损失。"""
        n = len(X)
        batch_size = batch_size or n
        rng = random.Random(seed)
        vW = [zeros(len(self.W[l]), len(self.W[l][0])) for l in range(len(self.W))]
        vb = [ [0.0] * len(self.b[l]) for l in range(len(self.b)) ]
        history: List[float] = []

        for epoch in range(1, epochs + 1):
            order = list(range(n))
            if shuffle:
                rng.shuffle(order)
            epoch_loss = 0.0
            for start in range(0, n, batch_size):
                idxs = order[start:start + batch_size]
                m = len(idxs)
                gW = [zeros(len(self.W[l]), len(self.W[l][0])) for l in range(len(self.W))]
                gb = [ [0.0] * len(self.b[l]) for l in range(len(self.b)) ]
                for i in idxs:
                    J, dW, db = self.backward(X[i], Y[i])
                    epoch_loss += J
                    for l in range(len(self.W)):
                        gl, dl = gW[l], dW[l]
                        for r in range(len(gl)):
                            gr, dr = gl[r], dl[r]
                            for c in range(len(gr)):
                                gr[c] += dr[c]
                        gbl, dbl = gb[l], db[l]
                        for r in range(len(gbl)):
                            gbl[r] += dbl[r]
                # 平均 + 更新(带动量)
                for l in range(len(self.W)):
                    gl, vl, Wl = gW[l], vW[l], self.W[l]
                    for r in range(len(gl)):
                        for c in range(len(gl[r])):
                            g = gl[r][c] / m
                            vl[r][c] = momentum * vl[r][c] - lr * g
                            Wl[r][c] += vl[r][c]
                    gbl, vbl, bl = gb[l], vb[l], self.b[l]
                    for r in range(len(gbl)):
                        g = gbl[r] / m
                        vbl[r] = momentum * vbl[r] - lr * g
                        bl[r] += vbl[r]
            history.append(epoch_loss / n)
            if verbose and (epoch % log_every == 0 or epoch == 1):
                print(f"epoch {epoch:5d}   loss = {history[-1]:.6e}")
        return history

    # ---------------- 参数扁平化:梯度检查用 ----------------
    def get_params(self) -> Vector:
        flat: Vector = []
        for W in self.W:
            for row in W:
                flat.extend(row)
        for b in self.b:
            flat.extend(b)
        return flat

    def set_params(self, flat: Vector) -> None:
        k = 0
        for W in self.W:
            for r in range(len(W)):
                for c in range(len(W[r])):
                    W[r][c] = flat[k]
                    k += 1
        for b in self.b:
            for r in range(len(b)):
                b[r] = flat[k]
                k += 1

    def n_params(self) -> int:
        return len(self.get_params())


# ---------------------------------------------------------------------------
# 4. 梯度检查:解析梯度 vs 有限差分
# ---------------------------------------------------------------------------

def numeric_gradient(
    net: MLP, X: Sequence[Vector], Y: Sequence[Vector], eps: float = 1e-6
) -> Vector:
    """用中心差分估计 d(平均损失)/d(theta)。"""
    flat = net.get_params()
    grads: Vector = []
    m = len(X)
    for i in range(len(flat)):
        original = flat[i]
        flat[i] = original + eps
        net.set_params(flat)
        plus = sum(net.loss_on(X[k], Y[k]) for k in range(m)) / m
        flat[i] = original - eps
        net.set_params(flat)
        minus = sum(net.loss_on(X[k], Y[k]) for k in range(m)) / m
        flat[i] = original
        grads.append((plus - minus) / (2.0 * eps))
    net.set_params(flat)
    return grads


def analytic_gradient(net: MLP, X: Sequence[Vector], Y: Sequence[Vector]) -> Vector:
    """把 backward 的解析梯度按与 get_params 相同的顺序展平。"""
    m = len(X)
    gW = [zeros(len(net.W[l]), len(net.W[l][0])) for l in range(len(net.W))]
    gb = [[0.0] * len(net.b[l]) for l in range(len(net.b))]
    for k in range(m):
        _, dW, db = net.backward(X[k], Y[k])
        for l in range(len(net.W)):
            for r in range(len(dW[l])):
                for c in range(len(dW[l][r])):
                    gW[l][r][c] += dW[l][r][c]
            for r in range(len(db[l])):
                gb[l][r] += db[l][r]
    flat: Vector = []
    for l in range(len(gW)):
        for row in gW[l]:
            flat.extend(v / m for v in row)
    for l in range(len(gb)):
        flat.extend(v / m for v in gb[l])
    return flat


def check_gradient(
    net: MLP,
    X: Sequence[Vector],
    Y: Sequence[Vector],
    eps: float = 1e-6,
) -> float:
    """返回解析梯度与数值梯度的最大相对误差。"""
    ana = analytic_gradient(net, X, Y)
    num = numeric_gradient(net, X, Y, eps=eps)
    worst = 0.0
    for a, n in zip(ana, num):
        denom = max(1e-8, abs(a) + abs(n))
        worst = max(worst, abs(a - n) / denom)
    return worst

《原子智能》· 纸质出版预备版 · PolyAI Team 著