Appearance
linear_unit.py
复制以下内容并保存为 linear_unit.py,使用方法见配套代码说明。
python
# ===========================================================================
# 代码清单 2-1 基础向量运算与回归评价指标(模块 linear_unit,第 1 部分)
# ===========================================================================
import math
import random
# ---------------------------------------------------------------------------
# (1) 基础工具函数
# ---------------------------------------------------------------------------
def dot(a, b):
"""向量点积 w·x,对应数学式 sum_i w_i x_i。"""
total = 0.0
for ai, bi in zip(a, b):
total += ai * bi
return total
def axpy(alpha, x, y):
"""返回 alpha*x + y,即 "a times x plus y",BLAS 的一级操作。"""
return [alpha * xi + yi for xi, yi in zip(x, y)]
def zeros(n):
"""长度为 n 的全零向量。"""
return [0.0] * n
def mean(values):
return sum(values) / len(values)
def std(values, ddof=0):
"""标准差。ddof=0 为总体标准差(除以 n);ddof=1 为样本标准差(除以 n-1)。"""
if len(values) <= ddof:
return 0.0
mu = mean(values)
var = sum((v - mu) ** 2 for v in values) / (len(values) - ddof)
return math.sqrt(var)
def col(X, j):
"""取出矩阵 X 的第 j 列。X 是 list[list[float]],行是样本,列是特征。"""
return [row[j] for row in X]
def shape(X):
"""返回 (样本数 m, 特征数 n)。"""
return (len(X), len(X[0]) if X else 0)
# ---------------------------------------------------------------------------
# (2) 回归评价指标:MSE、RMSE、MAE 与决定系数 R²
# ---------------------------------------------------------------------------
def mse(y_true, y_pred):
return sum((a - b) ** 2 for a, b in zip(y_true, y_pred)) / len(y_true)
def rmse(y_true, y_pred):
return math.sqrt(mse(y_true, y_pred))
def mae(y_true, y_pred):
return sum(abs(a - b) for a, b in zip(y_true, y_pred)) / len(y_true)
def r2_score(y_true, y_pred):
y_bar = mean(y_true)
ss_res = sum((a - b) ** 2 for a, b in zip(y_true, y_pred))
ss_tot = sum((a - y_bar) ** 2 for a in y_true)
if ss_tot < 1e-300:
return 0.0
return 1.0 - ss_res / ss_tot
def regression_report(y_true, y_pred):
"""返回一个人类可读的回归报告字符串。"""
return ("R² = %8.4f\nRMSE = %8.4f\nMAE = %8.4f"
% (r2_score(y_true, y_pred), rmse(y_true, y_pred),
mae(y_true, y_pred)))
# ===========================================================================
# 代码清单 2-2 LinearUnit 类(模块 linear_unit,第 2 部分)
# 批量 / 小批量 / 随机梯度下降 + 动量 + L2 正则化 + 学习率调度
# ===========================================================================
class LinearUnit:
"""线性单元:y_hat = w·x + b,用梯度下降最小化均方误差。
损失函数(注意那个 1/2,它在求导时会掉下来):
J(w, b) = 1/(2m) * sum_i ( y_i - (w·x_i + b) )^2
梯度(推导见正文第 2.3.3 节式 (15)、(16)):
dJ/dw_j = -1/m * sum_i (y_i - y_hat_i) * x_ij
dJ/db = -1/m * sum_i (y_i - y_hat_i)
参数
----
lr : 学习率(步长)η
n_epochs : 遍历数据集的次数
mode : "batch" | "sgd" | "minibatch"
batch_size : minibatch 模式下的批大小
l2 : L2 正则化强度 λ(0 表示关闭)
momentum : 动量系数 β(0 表示关闭),对应正文式 (22)
lr_schedule : "constant"(固定学习率)| "inverse"(逆时衰减)| "step"(阶梯衰减)
tol : 全量梯度范数小于 tol 时判定收敛并提前停止
seed : 随机种子,保证结果可复现
"""
def __init__(self, lr=0.01, n_epochs=200, mode="batch", batch_size=16,
l2=0.0, momentum=0.0, lr_schedule="constant", tol=1e-12,
seed=0, shuffle=True, record_params=False):
self.lr = lr
self.n_epochs = n_epochs
self.mode = mode
self.batch_size = batch_size
self.l2 = l2
self.momentum = momentum
self.lr_schedule = lr_schedule
self.tol = tol
self.seed = seed
self.shuffle = shuffle
self.record_params = record_params
self.w = None
self.b = 0.0
self.history = {"epoch": [], "loss": [], "grad_norm": [], "val_loss": []}
self.param_path = [] # record_params=True 时记录每一步的 (w, b)
self.n_features_ = None
self.n_iter_ = 0
self.converged_ = False
# -- 前向 --------------------------------------------------------------
def predict(self, X):
"""对一组样本做预测。"""
return [dot(self.w, x) + self.b for x in X]
def predict_one(self, x):
return dot(self.w, x) + self.b
# -- 损失与梯度 --------------------------------------------------------
def loss(self, X, y):
"""均方误差(含 L2 惩罚项,方便直接和梯度对照)。"""
m = len(X)
total = 0.0
for xi, yi in zip(X, y):
r = yi - (dot(self.w, xi) + self.b)
total += r * r
j = total / (2.0 * m)
if self.l2 > 0.0:
j += 0.5 * self.l2 * dot(self.w, self.w)
return j
def gradient(self, X, y):
"""返回 (dJ/dw, dJ/db)。
这是整个教程的核心。注意代码里的两行累加,
就是把教程中的公式逐个 for 循环写出来。
"""
m = len(X)
gw = zeros(self.n_features_)
gb = 0.0
for xi, yi in zip(X, y):
r = yi - (dot(self.w, xi) + self.b) # 残差 r = y - y_hat
for j in range(self.n_features_):
gw[j] -= r * xi[j] # 累加 -(y - y_hat) * x_j
gb -= r
# 取平均,再补上 L2 的梯度 λw
for j in range(self.n_features_):
gw[j] = gw[j] / m + self.l2 * self.w[j]
gb = gb / m
return gw, gb
# -- 训练 --------------------------------------------------------------
def _learning_rate(self, epoch):
"""学习率调度:固定学习率,或两种常用的衰减方式。"""
if self.lr_schedule == "constant":
return self.lr
if self.lr_schedule == "inverse":
# η_t = η_0 / (1 + t/T),温和地衰减
return self.lr / (1.0 + epoch / max(1.0, self.n_epochs / 4.0))
if self.lr_schedule == "step":
# 每 1/3 个训练过程砍半,像阶梯一样
span = max(1, self.n_epochs // 3)
drops = epoch // span
return self.lr / (2.0 ** drops)
raise ValueError("未知的 lr_schedule: %r" % (self.lr_schedule,))
def fit(self, X, y, X_val=None, y_val=None, verbose=False):
"""训练。X 是 list[list[float]],y 是 list[float]。"""
m, n = shape(X)
self.n_features_ = n
self.w = zeros(n)
self.b = 0.0
self.history = {"epoch": [], "loss": [], "grad_norm": [], "val_loss": []}
self.param_path = []
self.converged_ = False
rng = random.Random(self.seed)
state = (zeros(n), 0.0) # 动量缓存 (vw, vb)
for epoch in range(self.n_epochs):
eta = self._learning_rate(epoch)
if self.mode == "batch":
gw, gb = self.gradient(X, y)
state = self._apply_update(gw, gb, eta, state)
else:
state = self._stochastic_pass(X, y, rng, eta, state)
# 记录(注意:记录时重新算一遍真实的全量损失,方便画图)
gw_full, gb_full = self.gradient(X, y)
gnorm = math.sqrt(dot(gw_full, gw_full) + gb_full * gb_full)
train_loss = self.loss(X, y)
self.history["epoch"].append(epoch)
self.history["loss"].append(train_loss)
self.history["grad_norm"].append(gnorm)
if X_val is not None:
self.history["val_loss"].append(self.loss(X_val, y_val))
self.n_iter_ = epoch + 1
if self.record_params:
self.param_path.append((list(self.w), self.b))
if verbose and (epoch % max(1, self.n_epochs // 10) == 0):
print("epoch %4d loss=%.6f |grad|=%.3e eta=%.4f"
% (epoch, train_loss, gnorm, eta))
# 收敛判据:全量梯度的范数足够小
if gnorm < self.tol:
self.converged_ = True
break
return self
def _apply_update(self, gw, gb, eta, state):
"""执行一次参数更新,返回新的动量缓存。批量与随机模式共用这一份代码。
无动量:w ← w - η g
有动量:v ← β v - η g ; w ← w + v (即正文式 (22))
"""
n = self.n_features_
if self.momentum > 0.0:
vw, vb = state
vw = [self.momentum * vw[j] - eta * gw[j] for j in range(n)]
vb = self.momentum * vb - eta * gb
self.w = [self.w[j] + vw[j] for j in range(n)]
self.b = self.b + vb
return (vw, vb)
self.w = [self.w[j] - eta * gw[j] for j in range(n)]
self.b = self.b - eta * gb
return state
def _stochastic_pass(self, X, y, rng, eta, state):
"""SGD / mini-batch:在本 epoch 内把数据集切成若干批,每批更新一次。
同样是"一个 epoch",随机梯度下降其实做了 m 次(或 m/B 次)参数更新,
所以它每一步都更便宜、但走的路更抖。
"""
m = len(X)
idx = list(range(m))
if self.shuffle:
rng.shuffle(idx)
size = 1 if self.mode == "sgd" else max(1, self.batch_size)
start = 0
while start < m:
batch_idx = idx[start:start + size]
bx = [X[i] for i in batch_idx]
by = [y[i] for i in batch_idx]
gw, gb = self._batch_gradient(bx, by)
state = self._apply_update(gw, gb, eta, state)
start += size
return state
def _batch_gradient(self, bx, by):
"""在一个小批量上算梯度(和全量公式一样,只是 m 变成了 batch_size)。"""
m = len(bx)
gw = zeros(self.n_features_)
gb = 0.0
for xi, yi in zip(bx, by):
r = yi - (dot(self.w, xi) + self.b)
for j in range(self.n_features_):
gw[j] -= r * xi[j]
gb -= r
for j in range(self.n_features_):
gw[j] = gw[j] / m + self.l2 * self.w[j]
return gw, gb / m
# -- 评估 --------------------------------------------------------------
def score(self, X, y):
"""决定系数 R²。1.0 是完美拟合,0.0 相当于直接预测 y 的均值。
R² = 1 - SS_res / SS_tot
"""
y_hat = self.predict(X)
y_bar = mean(y)
ss_res = sum((yi - yhi) ** 2 for yi, yhi in zip(y, y_hat))
ss_tot = sum((yi - y_bar) ** 2 for yi in y)
if ss_tot < 1e-300:
return 0.0
return 1.0 - ss_res / ss_tot
# ===========================================================================
# 代码清单 2-3 特征缩放、多项式特征、正规方程、梯度校验、条件数与数据划分
# (模块 linear_unit,第 3 部分)
# ===========================================================================
# ---------------------------------------------------------------------------
# (1) 特征缩放
# ---------------------------------------------------------------------------
class StandardScaler:
"""把每一列变成均值 0、标准差 1。
对梯度下降而言,它往往决定了"能否在合理步数内收敛"。
化学描述符的量纲差异极大(电负性 ~2,原子质量 ~100,熔点 ~1000 K),
不做标准化时,损失函数的等高线是一根被抽长的雪茄,
梯度下降可能在较陡的方向上反复调整(详见正文第 2.3.2 节)。
"""
def __init__(self):
self.mean_ = None
self.std_ = None
def fit(self, X):
n = len(X[0])
self.mean_ = [mean(col(X, j)) for j in range(n)]
self.std_ = [std(col(X, j), ddof=0) for j in range(n)]
# 常数列(方差为 0)不能除以 0,保持原样
for j in range(n):
if self.std_[j] < 1e-12:
self.std_[j] = 1.0
return self
def transform(self, X):
out = []
for row in X:
out.append([(row[j] - self.mean_[j]) / self.std_[j]
for j in range(len(row))])
return out
def fit_transform(self, X):
return self.fit(X).transform(X)
def inverse_transform(self, X):
out = []
for row in X:
out.append([row[j] * self.std_[j] + self.mean_[j]
for j in range(len(row))])
return out
class MinMaxScaler:
"""把每一列线性映射到 [0, 1]。对异常值比 StandardScaler 敏感。"""
def __init__(self):
self.min_ = None
self.max_ = None
def fit(self, X):
n = len(X[0])
self.min_ = [min(col(X, j)) for j in range(n)]
self.max_ = [max(col(X, j)) for j in range(n)]
return self
def transform(self, X):
out = []
for row in X:
new_row = []
for j in range(len(row)):
span = self.max_[j] - self.min_[j]
if span < 1e-12:
span = 1.0
new_row.append((row[j] - self.min_[j]) / span)
out.append(new_row)
return out
def fit_transform(self, X):
return self.fit(X).transform(X)
# ---------------------------------------------------------------------------
# (2) 多项式特征
# ---------------------------------------------------------------------------
class PolynomialFeatures:
"""生成多项式特征。degree=2 时把 [x1, x2] 变成 [x1, x2, x1², x2², x1x2]。
用途:当响应随某个变量弯曲时,可以加入平方、交叉等项来表达曲率。
注意:模型对参数 w 仍然是线性的,因此仍属于"线性模型";
但多项式在训练区间之外往往表现很差(见正文第 2.5.3 节)。
"""
def __init__(self, degree=2, include_interactions=True):
self.degree = degree
self.include_interactions = include_interactions
def fit(self, X, y=None):
return self
def transform(self, X):
out = []
for row in X:
new_row = list(row)
n = len(row)
if self.degree >= 2:
for j in range(n):
new_row.append(row[j] ** 2)
if self.include_interactions:
for j in range(n):
for k in range(j + 1, n):
new_row.append(row[j] * row[k])
if self.degree >= 3:
for j in range(n):
new_row.append(row[j] ** 3)
out.append(new_row)
return out
def fit_transform(self, X, y=None):
return self.transform(X)
# ---------------------------------------------------------------------------
# (3) 正规方程(手写高斯消元,用来与梯度下降的结果相互核对)
# ---------------------------------------------------------------------------
def solve_linear_system(A, b):
"""高斯消元 + 部分主元,解 A x = b。A 是 n×n,b 是长度 n 的向量。"""
n = len(A)
M = [list(A[i]) + [b[i]] for i in range(n)]
for i in range(n):
# 选主元,避免除以一个很小的数
pivot = max(range(i, n), key=lambda r: abs(M[r][i]))
if abs(M[pivot][i]) < 1e-300:
raise ValueError("矩阵奇异,无法求解")
M[i], M[pivot] = M[pivot], M[i]
for r in range(i + 1, n):
factor = M[r][i] / M[i][i]
for c in range(i, n + 1):
M[r][c] -= factor * M[i][c]
x = [0.0] * n
for i in range(n - 1, -1, -1):
s = M[i][n] - sum(M[i][c] * x[c] for c in range(i + 1, n))
x[i] = s / M[i][i]
return x
def normal_equation(X, y, fit_intercept=True):
"""最小二乘的解析解:w = (XᵀX)⁻¹ Xᵀy,用增广列 1 吸收截距。
返回 (w, b)。它一次求出结果,不需要学习率,也不需要迭代,
但有两个前提:XᵀX 可逆(特征之间没有严格的线性相关),且特征数不能太大。
注意:这里的增广列 1 放在第 0 列,与正文把它放在最后一列只是排列不同。
工程实践中通常改用 QR 分解或 Cholesky 分解,数值上更稳定。
"""
if fit_intercept:
Xa = [[1.0] + list(row) for row in X]
else:
Xa = [list(row) for row in X]
m, n = shape(Xa)
# A = XᵀX, c = Xᵀy
A = [[sum(Xa[i][p] * Xa[i][q] for i in range(m)) for q in range(n)]
for p in range(n)]
c = [sum(Xa[i][p] * y[i] for i in range(m)) for p in range(n)]
theta = solve_linear_system(A, c)
if fit_intercept:
return theta[1:], theta[0]
return theta, 0.0
# ---------------------------------------------------------------------------
# (4) 数值梯度校验(中心差分)
# ---------------------------------------------------------------------------
def numerical_gradient(f, theta, h=1e-6):
"""中心差分:(f(x+h) - f(x-h)) / 2h。精度 O(h²)。
用它来校验手写的解析梯度。如果两者对不上,
最常见的原因是求导时漏了负号或漏除了样本数 m。
"""
grad = zeros(len(theta))
for i in range(len(theta)):
orig = theta[i]
theta[i] = orig + h
f_plus = f(theta)
theta[i] = orig - h
f_minus = f(theta)
theta[i] = orig
grad[i] = (f_plus - f_minus) / (2.0 * h)
return grad
def check_gradient(X, y, theta, h=1e-6):
"""把 (w, b) 拼成一个向量 theta,比较解析梯度与数值梯度。
返回 (解析梯度, 数值梯度, 最大相对误差)。
"""
n = len(X[0])
unit = LinearUnit()
unit.n_features_ = n
unit.w = theta[:n]
unit.b = theta[n] if len(theta) > n else 0.0
def f(th):
unit.w = th[:n]
unit.b = th[n] if len(th) > n else 0.0
return unit.loss(X, y)
gw, gb = unit.gradient(X, y)
analytic = list(gw) + [gb]
numeric = numerical_gradient(f, list(theta), h)
max_rel = 0.0
for a, b in zip(analytic, numeric):
denom = max(1e-12, abs(a) + abs(b))
max_rel = max(max_rel, abs(a - b) / denom)
return analytic, numeric, max_rel
# ---------------------------------------------------------------------------
# (5) 条件数估计(幂迭代求 λmax,反幂迭代求 λmin)
# ---------------------------------------------------------------------------
def condition_number(X, n_iter=200, fit_intercept=True):
"""估计 A = XᵀX/m 的条件数 κ = λmax / λmin。
λmax 用幂迭代求,λmin 用反幂迭代求(每步解一次线性方程组,
正好复用 normal_equation 用到的那个高斯消元)。
fit_intercept=True 时会把一列 1 拼到 X 前面——这一列代表截距项 b。
这一点很关键:梯度下降真正面对的是增广矩阵的 Hessian。
若某个特征的均值远大于它的标准差(例如 1000/T ≈ 3.15 ± 0.14),
它与常数列 1 几乎平行,条件数会很大——即使它的数值量级"正常"。
κ = 1 时损失等高线是正圆,取 η = 1/λ 一步即可到达极小点;
κ = 1000 时等高线是极扁的椭圆,梯度下降会在窄方向上反复横跳。
"""
if fit_intercept:
X = [[1.0] + list(row) for row in X]
m, n = shape(X)
cols = [[row[j] for row in X] for j in range(n)]
# A = X^T X
A = [[sum(cols[p][i] * cols[q][i] for i in range(m)) / m
for q in range(n)] for p in range(n)]
def matvec(v):
return [sum(A[p][q] * v[q] for q in range(n)) for p in range(n)]
def power_iteration():
"""最大特征值:v ← A v / |A v|"""
v = [1.0] + [0.0] * (n - 1)
lam = 0.0
for _ in range(n_iter):
w = matvec(v)
norm = math.sqrt(dot(w, w))
if norm < 1e-300:
return 0.0
v = [wi / norm for wi in w]
lam = dot(v, matvec(v))
return lam
def inverse_power_iteration(lam_max):
"""最小特征值:解 A z = v 再归一化(即对 A⁻¹ 做幂迭代)。"""
# 加一个很小的 ridge,防止 A 数值奇异
shift = 1e-12 * (lam_max + 1.0)
As = [[A[i][j] + (shift if i == j else 0.0) for j in range(n)]
for i in range(n)]
v = [1.0] + [0.0] * (n - 1)
lam = 0.0
for _ in range(n_iter):
z = solve_linear_system(As, v)
norm = math.sqrt(dot(z, z))
if norm < 1e-300:
return 0.0
v = [zi / norm for zi in z]
lam = dot(v, matvec(v))
return lam
lam_max = power_iteration()
if n == 1:
lam_min = lam_max
else:
lam_min = inverse_power_iteration(lam_max)
lam_min = max(lam_min, 1e-12)
return lam_max / lam_min
# ---------------------------------------------------------------------------
# (6) 数据划分(支持按"化学体系"分组划分)
# ---------------------------------------------------------------------------
def train_test_split(X, y, test_size=0.3, seed=0, groups=None):
"""划分数据集。
groups 不为 None 时按组划分:同一组(例如同一个化学体系)只会出现在
训练集或测试集的一侧。随机划分会让高度相似的同族样本同时出现在
训练集和测试集中,使测试误差偏乐观(见正文第 2.5.2 节)。
说明:按组划分时测试集大小只能以"整组"为单位逼近 test_size。
"""
m = len(X)
idx = list(range(m))
rng = random.Random(seed)
rng.shuffle(idx)
n_test = int(round(m * test_size))
if groups is None:
test_idx = idx[:n_test]
train_idx = idx[n_test:]
else:
# 按组打散后整组分配
unique_groups = []
for g in groups:
if g not in unique_groups:
unique_groups.append(g)
rng.shuffle(unique_groups)
chosen = set()
count = 0
for g in unique_groups:
if count >= n_test:
break
chosen.add(g)
count += sum(1 for gi in groups if gi == g)
test_idx = [i for i in idx if groups[i] in chosen]
train_idx = [i for i in idx if groups[i] not in chosen]
X_train = [X[i] for i in train_idx]
y_train = [y[i] for i in train_idx]
X_test = [X[i] for i in test_idx]
y_test = [y[i] for i in test_idx]
return X_train, X_test, y_train, y_test
def kfold_indices(m, k=5, seed=0):
"""生成 k 折交叉验证的索引,返回 [(train_idx, test_idx), ...]。"""
idx = list(range(m))
random.Random(seed).shuffle(idx)
folds = []
for fold in range(k):
test_idx = [idx[i] for i in range(m) if i % k == fold]
train_idx = [idx[i] for i in range(m) if i % k != fold]
folds.append((train_idx, test_idx))
return folds
# ---------------------------------------------------------------------------
# (7) 便捷封装函数
# ---------------------------------------------------------------------------
def fit_linear_unit(X, y, lr=0.05, n_epochs=500, scale=True, mode="batch",
**kwargs):
"""标准化 → 训练 → 返回 (模型, scaler)。
标准化空间中的权重表示"特征变化一个标准差时响应的变化量";
若要得到原始量纲下的斜率,用 w_j / std_j 还原(见正文第 2.5.2.4 节)。
"""
scaler = StandardScaler()
Xs = scaler.fit_transform(X) if scale else X
model = LinearUnit(lr=lr, n_epochs=n_epochs, mode=mode, **kwargs)
model.fit(Xs, y)
model.scaler_ = scaler if scale else None
model.X_mean_ = scaler.mean_ if scale else None
return model, scaler
def predict_raw(model, X):
"""用训练时保存的 scaler 对新数据做预测,自动处理标准化。"""
scaler = getattr(model, "scaler_", None)
Xs = scaler.transform(X) if scaler is not None else X
return model.predict(Xs)