图片详情
—
Skip to content

linear_unit.py ​

返回章节 · 配套代码说明

复制以下内容并保存为 linear_unit.py,使用方法见配套代码说明。

python
# ===========================================================================
# 代码清单 2-1  基础向量运算与回归评价指标(模块 linear_unit,第 1 部分)
# ===========================================================================
import math
import random


# ---------------------------------------------------------------------------
# (1) 基础工具函数
# ---------------------------------------------------------------------------
def dot(a, b):
    """向量点积 w·x,对应数学式 sum_i w_i x_i。"""
    total = 0.0
    for ai, bi in zip(a, b):
        total += ai * bi
    return total


def axpy(alpha, x, y):
    """返回 alpha*x + y,即 "a times x plus y",BLAS 的一级操作。"""
    return [alpha * xi + yi for xi, yi in zip(x, y)]


def zeros(n):
    """长度为 n 的全零向量。"""
    return [0.0] * n


def mean(values):
    return sum(values) / len(values)


def std(values, ddof=0):
    """标准差。ddof=0 为总体标准差(除以 n);ddof=1 为样本标准差(除以 n-1)。"""
    if len(values) <= ddof:
        return 0.0
    mu = mean(values)
    var = sum((v - mu) ** 2 for v in values) / (len(values) - ddof)
    return math.sqrt(var)


def col(X, j):
    """取出矩阵 X 的第 j 列。X 是 list[list[float]],行是样本,列是特征。"""
    return [row[j] for row in X]


def shape(X):
    """返回 (样本数 m, 特征数 n)。"""
    return (len(X), len(X[0]) if X else 0)




# ---------------------------------------------------------------------------
# (2) 回归评价指标:MSE、RMSE、MAE 与决定系数 R²
# ---------------------------------------------------------------------------
def mse(y_true, y_pred):
    return sum((a - b) ** 2 for a, b in zip(y_true, y_pred)) / len(y_true)


def rmse(y_true, y_pred):
    return math.sqrt(mse(y_true, y_pred))


def mae(y_true, y_pred):
    return sum(abs(a - b) for a, b in zip(y_true, y_pred)) / len(y_true)


def r2_score(y_true, y_pred):
    y_bar = mean(y_true)
    ss_res = sum((a - b) ** 2 for a, b in zip(y_true, y_pred))
    ss_tot = sum((a - y_bar) ** 2 for a in y_true)
    if ss_tot < 1e-300:
        return 0.0
    return 1.0 - ss_res / ss_tot


def regression_report(y_true, y_pred):
    """返回一个人类可读的回归报告字符串。"""
    return ("R²   = %8.4f\nRMSE = %8.4f\nMAE  = %8.4f"
            % (r2_score(y_true, y_pred), rmse(y_true, y_pred),
               mae(y_true, y_pred)))

# ===========================================================================
# 代码清单 2-2  LinearUnit 类(模块 linear_unit,第 2 部分)
#   批量 / 小批量 / 随机梯度下降 + 动量 + L2 正则化 + 学习率调度
# ===========================================================================
class LinearUnit:
    """线性单元:y_hat = w·x + b,用梯度下降最小化均方误差。

    损失函数(注意那个 1/2,它在求导时会掉下来):

        J(w, b) = 1/(2m) * sum_i ( y_i - (w·x_i + b) )^2

    梯度(推导见正文第 2.3.3 节式 (15)、(16)):

        dJ/dw_j = -1/m * sum_i (y_i - y_hat_i) * x_ij
        dJ/db   = -1/m * sum_i (y_i - y_hat_i)

    参数
    ----
    lr           : 学习率(步长)η
    n_epochs     : 遍历数据集的次数
    mode         : "batch" | "sgd" | "minibatch"
    batch_size   : minibatch 模式下的批大小
    l2           : L2 正则化强度 λ(0 表示关闭)
    momentum     : 动量系数 β(0 表示关闭),对应正文式 (22)
    lr_schedule  : "constant"(固定学习率)| "inverse"(逆时衰减)| "step"(阶梯衰减)
    tol          : 全量梯度范数小于 tol 时判定收敛并提前停止
    seed         : 随机种子,保证结果可复现
    """

    def __init__(self, lr=0.01, n_epochs=200, mode="batch", batch_size=16,
                 l2=0.0, momentum=0.0, lr_schedule="constant", tol=1e-12,
                 seed=0, shuffle=True, record_params=False):
        self.lr = lr
        self.n_epochs = n_epochs
        self.mode = mode
        self.batch_size = batch_size
        self.l2 = l2
        self.momentum = momentum
        self.lr_schedule = lr_schedule
        self.tol = tol
        self.seed = seed
        self.shuffle = shuffle
        self.record_params = record_params

        self.w = None
        self.b = 0.0
        self.history = {"epoch": [], "loss": [], "grad_norm": [], "val_loss": []}
        self.param_path = []          # record_params=True 时记录每一步的 (w, b)
        self.n_features_ = None
        self.n_iter_ = 0
        self.converged_ = False

    # -- 前向 --------------------------------------------------------------

    def predict(self, X):
        """对一组样本做预测。"""
        return [dot(self.w, x) + self.b for x in X]

    def predict_one(self, x):
        return dot(self.w, x) + self.b

    # -- 损失与梯度 --------------------------------------------------------

    def loss(self, X, y):
        """均方误差(含 L2 惩罚项,方便直接和梯度对照)。"""
        m = len(X)
        total = 0.0
        for xi, yi in zip(X, y):
            r = yi - (dot(self.w, xi) + self.b)
            total += r * r
        j = total / (2.0 * m)
        if self.l2 > 0.0:
            j += 0.5 * self.l2 * dot(self.w, self.w)
        return j

    def gradient(self, X, y):
        """返回 (dJ/dw, dJ/db)。

        这是整个教程的核心。注意代码里的两行累加,
        就是把教程中的公式逐个 for 循环写出来。
        """
        m = len(X)
        gw = zeros(self.n_features_)
        gb = 0.0
        for xi, yi in zip(X, y):
            r = yi - (dot(self.w, xi) + self.b)   # 残差 r = y - y_hat
            for j in range(self.n_features_):
                gw[j] -= r * xi[j]                # 累加 -(y - y_hat) * x_j
            gb -= r
        # 取平均,再补上 L2 的梯度 λw
        for j in range(self.n_features_):
            gw[j] = gw[j] / m + self.l2 * self.w[j]
        gb = gb / m
        return gw, gb

    # -- 训练 --------------------------------------------------------------

    def _learning_rate(self, epoch):
        """学习率调度:固定学习率,或两种常用的衰减方式。"""
        if self.lr_schedule == "constant":
            return self.lr
        if self.lr_schedule == "inverse":
            # η_t = η_0 / (1 + t/T),温和地衰减
            return self.lr / (1.0 + epoch / max(1.0, self.n_epochs / 4.0))
        if self.lr_schedule == "step":
            # 每 1/3 个训练过程砍半,像阶梯一样
            span = max(1, self.n_epochs // 3)
            drops = epoch // span
            return self.lr / (2.0 ** drops)
        raise ValueError("未知的 lr_schedule: %r" % (self.lr_schedule,))

    def fit(self, X, y, X_val=None, y_val=None, verbose=False):
        """训练。X 是 list[list[float]],y 是 list[float]。"""
        m, n = shape(X)
        self.n_features_ = n
        self.w = zeros(n)
        self.b = 0.0
        self.history = {"epoch": [], "loss": [], "grad_norm": [], "val_loss": []}
        self.param_path = []
        self.converged_ = False

        rng = random.Random(self.seed)
        state = (zeros(n), 0.0)     # 动量缓存 (vw, vb)

        for epoch in range(self.n_epochs):
            eta = self._learning_rate(epoch)

            if self.mode == "batch":
                gw, gb = self.gradient(X, y)
                state = self._apply_update(gw, gb, eta, state)
            else:
                state = self._stochastic_pass(X, y, rng, eta, state)

            # 记录(注意:记录时重新算一遍真实的全量损失,方便画图)
            gw_full, gb_full = self.gradient(X, y)
            gnorm = math.sqrt(dot(gw_full, gw_full) + gb_full * gb_full)
            train_loss = self.loss(X, y)
            self.history["epoch"].append(epoch)
            self.history["loss"].append(train_loss)
            self.history["grad_norm"].append(gnorm)
            if X_val is not None:
                self.history["val_loss"].append(self.loss(X_val, y_val))
            self.n_iter_ = epoch + 1
            if self.record_params:
                self.param_path.append((list(self.w), self.b))

            if verbose and (epoch % max(1, self.n_epochs // 10) == 0):
                print("epoch %4d  loss=%.6f  |grad|=%.3e  eta=%.4f"
                      % (epoch, train_loss, gnorm, eta))

            # 收敛判据:全量梯度的范数足够小
            if gnorm < self.tol:
                self.converged_ = True
                break

        return self

    def _apply_update(self, gw, gb, eta, state):
        """执行一次参数更新,返回新的动量缓存。批量与随机模式共用这一份代码。

        无动量:w ← w - η g
        有动量:v ← β v - η g ; w ← w + v      (即正文式 (22))
        """
        n = self.n_features_
        if self.momentum > 0.0:
            vw, vb = state
            vw = [self.momentum * vw[j] - eta * gw[j] for j in range(n)]
            vb = self.momentum * vb - eta * gb
            self.w = [self.w[j] + vw[j] for j in range(n)]
            self.b = self.b + vb
            return (vw, vb)
        self.w = [self.w[j] - eta * gw[j] for j in range(n)]
        self.b = self.b - eta * gb
        return state

    def _stochastic_pass(self, X, y, rng, eta, state):
        """SGD / mini-batch:在本 epoch 内把数据集切成若干批,每批更新一次。

        同样是"一个 epoch",随机梯度下降其实做了 m 次(或 m/B 次)参数更新,
        所以它每一步都更便宜、但走的路更抖。
        """
        m = len(X)
        idx = list(range(m))
        if self.shuffle:
            rng.shuffle(idx)
        size = 1 if self.mode == "sgd" else max(1, self.batch_size)

        start = 0
        while start < m:
            batch_idx = idx[start:start + size]
            bx = [X[i] for i in batch_idx]
            by = [y[i] for i in batch_idx]
            gw, gb = self._batch_gradient(bx, by)
            state = self._apply_update(gw, gb, eta, state)
            start += size
        return state

    def _batch_gradient(self, bx, by):
        """在一个小批量上算梯度(和全量公式一样,只是 m 变成了 batch_size)。"""
        m = len(bx)
        gw = zeros(self.n_features_)
        gb = 0.0
        for xi, yi in zip(bx, by):
            r = yi - (dot(self.w, xi) + self.b)
            for j in range(self.n_features_):
                gw[j] -= r * xi[j]
            gb -= r
        for j in range(self.n_features_):
            gw[j] = gw[j] / m + self.l2 * self.w[j]
        return gw, gb / m

    # -- 评估 --------------------------------------------------------------

    def score(self, X, y):
        """决定系数 R²。1.0 是完美拟合,0.0 相当于直接预测 y 的均值。

        R² = 1 - SS_res / SS_tot
        """
        y_hat = self.predict(X)
        y_bar = mean(y)
        ss_res = sum((yi - yhi) ** 2 for yi, yhi in zip(y, y_hat))
        ss_tot = sum((yi - y_bar) ** 2 for yi in y)
        if ss_tot < 1e-300:
            return 0.0
        return 1.0 - ss_res / ss_tot

# ===========================================================================
# 代码清单 2-3  特征缩放、多项式特征、正规方程、梯度校验、条件数与数据划分
#               (模块 linear_unit,第 3 部分)
# ===========================================================================
# ---------------------------------------------------------------------------
# (1) 特征缩放
# ---------------------------------------------------------------------------
class StandardScaler:
    """把每一列变成均值 0、标准差 1。

    对梯度下降而言,它往往决定了"能否在合理步数内收敛"。
    化学描述符的量纲差异极大(电负性 ~2,原子质量 ~100,熔点 ~1000 K),
    不做标准化时,损失函数的等高线是一根被抽长的雪茄,
    梯度下降可能在较陡的方向上反复调整(详见正文第 2.3.2 节)。
    """

    def __init__(self):
        self.mean_ = None
        self.std_ = None

    def fit(self, X):
        n = len(X[0])
        self.mean_ = [mean(col(X, j)) for j in range(n)]
        self.std_ = [std(col(X, j), ddof=0) for j in range(n)]
        # 常数列(方差为 0)不能除以 0,保持原样
        for j in range(n):
            if self.std_[j] < 1e-12:
                self.std_[j] = 1.0
        return self

    def transform(self, X):
        out = []
        for row in X:
            out.append([(row[j] - self.mean_[j]) / self.std_[j]
                        for j in range(len(row))])
        return out

    def fit_transform(self, X):
        return self.fit(X).transform(X)

    def inverse_transform(self, X):
        out = []
        for row in X:
            out.append([row[j] * self.std_[j] + self.mean_[j]
                        for j in range(len(row))])
        return out


class MinMaxScaler:
    """把每一列线性映射到 [0, 1]。对异常值比 StandardScaler 敏感。"""

    def __init__(self):
        self.min_ = None
        self.max_ = None

    def fit(self, X):
        n = len(X[0])
        self.min_ = [min(col(X, j)) for j in range(n)]
        self.max_ = [max(col(X, j)) for j in range(n)]
        return self

    def transform(self, X):
        out = []
        for row in X:
            new_row = []
            for j in range(len(row)):
                span = self.max_[j] - self.min_[j]
                if span < 1e-12:
                    span = 1.0
                new_row.append((row[j] - self.min_[j]) / span)
            out.append(new_row)
        return out

    def fit_transform(self, X):
        return self.fit(X).transform(X)




# ---------------------------------------------------------------------------
# (2) 多项式特征
# ---------------------------------------------------------------------------
class PolynomialFeatures:
    """生成多项式特征。degree=2 时把 [x1, x2] 变成 [x1, x2, x1², x2², x1x2]。

    用途:当响应随某个变量弯曲时,可以加入平方、交叉等项来表达曲率。
    注意:模型对参数 w 仍然是线性的,因此仍属于"线性模型";
    但多项式在训练区间之外往往表现很差(见正文第 2.5.3 节)。
    """

    def __init__(self, degree=2, include_interactions=True):
        self.degree = degree
        self.include_interactions = include_interactions

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        out = []
        for row in X:
            new_row = list(row)
            n = len(row)
            if self.degree >= 2:
                for j in range(n):
                    new_row.append(row[j] ** 2)
                if self.include_interactions:
                    for j in range(n):
                        for k in range(j + 1, n):
                            new_row.append(row[j] * row[k])
            if self.degree >= 3:
                for j in range(n):
                    new_row.append(row[j] ** 3)
            out.append(new_row)
        return out

    def fit_transform(self, X, y=None):
        return self.transform(X)




# ---------------------------------------------------------------------------
# (3) 正规方程(手写高斯消元,用来与梯度下降的结果相互核对)
# ---------------------------------------------------------------------------
def solve_linear_system(A, b):
    """高斯消元 + 部分主元,解 A x = b。A 是 n×n,b 是长度 n 的向量。"""
    n = len(A)
    M = [list(A[i]) + [b[i]] for i in range(n)]
    for i in range(n):
        # 选主元,避免除以一个很小的数
        pivot = max(range(i, n), key=lambda r: abs(M[r][i]))
        if abs(M[pivot][i]) < 1e-300:
            raise ValueError("矩阵奇异,无法求解")
        M[i], M[pivot] = M[pivot], M[i]
        for r in range(i + 1, n):
            factor = M[r][i] / M[i][i]
            for c in range(i, n + 1):
                M[r][c] -= factor * M[i][c]
    x = [0.0] * n
    for i in range(n - 1, -1, -1):
        s = M[i][n] - sum(M[i][c] * x[c] for c in range(i + 1, n))
        x[i] = s / M[i][i]
    return x


def normal_equation(X, y, fit_intercept=True):
    """最小二乘的解析解:w = (XᵀX)⁻¹ Xᵀy,用增广列 1 吸收截距。

    返回 (w, b)。它一次求出结果,不需要学习率,也不需要迭代,
    但有两个前提:XᵀX 可逆(特征之间没有严格的线性相关),且特征数不能太大。
    注意:这里的增广列 1 放在第 0 列,与正文把它放在最后一列只是排列不同。
    工程实践中通常改用 QR 分解或 Cholesky 分解,数值上更稳定。
    """
    if fit_intercept:
        Xa = [[1.0] + list(row) for row in X]
    else:
        Xa = [list(row) for row in X]
    m, n = shape(Xa)
    # A = XᵀX, c = Xᵀy
    A = [[sum(Xa[i][p] * Xa[i][q] for i in range(m)) for q in range(n)]
         for p in range(n)]
    c = [sum(Xa[i][p] * y[i] for i in range(m)) for p in range(n)]
    theta = solve_linear_system(A, c)
    if fit_intercept:
        return theta[1:], theta[0]
    return theta, 0.0




# ---------------------------------------------------------------------------
# (4) 数值梯度校验(中心差分)
# ---------------------------------------------------------------------------
def numerical_gradient(f, theta, h=1e-6):
    """中心差分:(f(x+h) - f(x-h)) / 2h。精度 O(h²)。

    用它来校验手写的解析梯度。如果两者对不上,
    最常见的原因是求导时漏了负号或漏除了样本数 m。
    """
    grad = zeros(len(theta))
    for i in range(len(theta)):
        orig = theta[i]
        theta[i] = orig + h
        f_plus = f(theta)
        theta[i] = orig - h
        f_minus = f(theta)
        theta[i] = orig
        grad[i] = (f_plus - f_minus) / (2.0 * h)
    return grad


def check_gradient(X, y, theta, h=1e-6):
    """把 (w, b) 拼成一个向量 theta,比较解析梯度与数值梯度。

    返回 (解析梯度, 数值梯度, 最大相对误差)。
    """
    n = len(X[0])
    unit = LinearUnit()
    unit.n_features_ = n
    unit.w = theta[:n]
    unit.b = theta[n] if len(theta) > n else 0.0

    def f(th):
        unit.w = th[:n]
        unit.b = th[n] if len(th) > n else 0.0
        return unit.loss(X, y)

    gw, gb = unit.gradient(X, y)
    analytic = list(gw) + [gb]
    numeric = numerical_gradient(f, list(theta), h)
    max_rel = 0.0
    for a, b in zip(analytic, numeric):
        denom = max(1e-12, abs(a) + abs(b))
        max_rel = max(max_rel, abs(a - b) / denom)
    return analytic, numeric, max_rel




# ---------------------------------------------------------------------------
# (5) 条件数估计(幂迭代求 λmax,反幂迭代求 λmin)
# ---------------------------------------------------------------------------
def condition_number(X, n_iter=200, fit_intercept=True):
    """估计 A = XᵀX/m 的条件数 κ = λmax / λmin。

    λmax 用幂迭代求,λmin 用反幂迭代求(每步解一次线性方程组,
    正好复用 normal_equation 用到的那个高斯消元)。

    fit_intercept=True 时会把一列 1 拼到 X 前面——这一列代表截距项 b。
    这一点很关键:梯度下降真正面对的是增广矩阵的 Hessian。
    若某个特征的均值远大于它的标准差(例如 1000/T ≈ 3.15 ± 0.14),
    它与常数列 1 几乎平行,条件数会很大——即使它的数值量级"正常"。

    κ = 1    时损失等高线是正圆,取 η = 1/λ 一步即可到达极小点;
    κ = 1000 时等高线是极扁的椭圆,梯度下降会在窄方向上反复横跳。
    """
    if fit_intercept:
        X = [[1.0] + list(row) for row in X]
    m, n = shape(X)
    cols = [[row[j] for row in X] for j in range(n)]
    # A = X^T X
    A = [[sum(cols[p][i] * cols[q][i] for i in range(m)) / m
          for q in range(n)] for p in range(n)]

    def matvec(v):
        return [sum(A[p][q] * v[q] for q in range(n)) for p in range(n)]

    def power_iteration():
        """最大特征值:v ← A v / |A v|"""
        v = [1.0] + [0.0] * (n - 1)
        lam = 0.0
        for _ in range(n_iter):
            w = matvec(v)
            norm = math.sqrt(dot(w, w))
            if norm < 1e-300:
                return 0.0
            v = [wi / norm for wi in w]
            lam = dot(v, matvec(v))
        return lam

    def inverse_power_iteration(lam_max):
        """最小特征值:解 A z = v 再归一化(即对 A⁻¹ 做幂迭代)。"""
        # 加一个很小的 ridge,防止 A 数值奇异
        shift = 1e-12 * (lam_max + 1.0)
        As = [[A[i][j] + (shift if i == j else 0.0) for j in range(n)]
              for i in range(n)]
        v = [1.0] + [0.0] * (n - 1)
        lam = 0.0
        for _ in range(n_iter):
            z = solve_linear_system(As, v)
            norm = math.sqrt(dot(z, z))
            if norm < 1e-300:
                return 0.0
            v = [zi / norm for zi in z]
            lam = dot(v, matvec(v))
        return lam

    lam_max = power_iteration()
    if n == 1:
        lam_min = lam_max
    else:
        lam_min = inverse_power_iteration(lam_max)
    lam_min = max(lam_min, 1e-12)
    return lam_max / lam_min




# ---------------------------------------------------------------------------
# (6) 数据划分(支持按"化学体系"分组划分)
# ---------------------------------------------------------------------------
def train_test_split(X, y, test_size=0.3, seed=0, groups=None):
    """划分数据集。

    groups 不为 None 时按组划分:同一组(例如同一个化学体系)只会出现在
    训练集或测试集的一侧。随机划分会让高度相似的同族样本同时出现在
    训练集和测试集中,使测试误差偏乐观(见正文第 2.5.2 节)。
    说明:按组划分时测试集大小只能以"整组"为单位逼近 test_size。
    """
    m = len(X)
    idx = list(range(m))
    rng = random.Random(seed)
    rng.shuffle(idx)
    n_test = int(round(m * test_size))

    if groups is None:
        test_idx = idx[:n_test]
        train_idx = idx[n_test:]
    else:
        # 按组打散后整组分配
        unique_groups = []
        for g in groups:
            if g not in unique_groups:
                unique_groups.append(g)
        rng.shuffle(unique_groups)
        chosen = set()
        count = 0
        for g in unique_groups:
            if count >= n_test:
                break
            chosen.add(g)
            count += sum(1 for gi in groups if gi == g)
        test_idx = [i for i in idx if groups[i] in chosen]
        train_idx = [i for i in idx if groups[i] not in chosen]

    X_train = [X[i] for i in train_idx]
    y_train = [y[i] for i in train_idx]
    X_test = [X[i] for i in test_idx]
    y_test = [y[i] for i in test_idx]
    return X_train, X_test, y_train, y_test


def kfold_indices(m, k=5, seed=0):
    """生成 k 折交叉验证的索引,返回 [(train_idx, test_idx), ...]。"""
    idx = list(range(m))
    random.Random(seed).shuffle(idx)
    folds = []
    for fold in range(k):
        test_idx = [idx[i] for i in range(m) if i % k == fold]
        train_idx = [idx[i] for i in range(m) if i % k != fold]
        folds.append((train_idx, test_idx))
    return folds




# ---------------------------------------------------------------------------
# (7) 便捷封装函数
# ---------------------------------------------------------------------------
def fit_linear_unit(X, y, lr=0.05, n_epochs=500, scale=True, mode="batch",
                    **kwargs):
    """标准化 → 训练 → 返回 (模型, scaler)。

    标准化空间中的权重表示"特征变化一个标准差时响应的变化量";
    若要得到原始量纲下的斜率,用 w_j / std_j 还原(见正文第 2.5.2.4 节)。
    """
    scaler = StandardScaler()
    Xs = scaler.fit_transform(X) if scale else X
    model = LinearUnit(lr=lr, n_epochs=n_epochs, mode=mode, **kwargs)
    model.fit(Xs, y)
    model.scaler_ = scaler if scale else None
    model.X_mean_ = scaler.mean_ if scale else None
    return model, scaler


def predict_raw(model, X):
    """用训练时保存的 scaler 对新数据做预测,自动处理标准化。"""
    scaler = getattr(model, "scaler_", None)
    Xs = scaler.transform(X) if scaler is not None else X
    return model.predict(Xs)

《原子智能》· 纸质出版预备版 · PolyAI Team 著