对于前面已经提到的类及一些细节不再给出.对于稀疏矩阵的了解是必要的. from abc import abcmeta, abstractmethod import warnings import n mpy as np from scipy import linalg from scipy import sparse from scipy.sparse import linalg as sp_linalg from .base import linearclassifiermixin, linearmodel, _rescale_data from .sag import sag_solver :是一种随机平均梯度下降式的求解岭回归与logistic回归的包, 使用梯度下降法 这种算法收敛很快. from ..base import regressormixin from .. tils.extmath import safe_aparse_dot from .. tils.extmath import row_norms: 行范数,不支持稀疏矩阵. from .. tils import check_x_y from .. tils import check_array :转换为ndarray类型. from .. tils import check_consistent_length: 检查一个ndarray的list 是否所有元素第一个 维度都相等.(i.e. same length) from .. tils import col mn_or_1d: 特殊的拉直函数,接受类似feat re形式的ndarray, 即在第二个维度上只有1维,并将其按列拉直为以为数组. from ..preprocessing import labelbinarizer: 对一对多问题将标签进行二值化. from ..model_selection import gridsearchcv from ..externsls import six from ..metrics.scorer import check_scoring: 对能够进行返回score估计的模型,返回进行 score计算的函数. 下面先不对具体的类,而是接口进行说明.(无组织架构) sp_linalg.aslinearoperator: 将对象(ndarray, sparse, matrix an so on)转换为线性算子. np.empty(shape): 返回相应shape的未初始化的ndarray. sp_linalg.cg: 使用共轭梯度(conj gate gradient)法解线性系统. sq_linalg.lsqr: lsqr :qr分解. ndarray.flat:(...) np.atleast_1d: 标量化为一维数组,高维保持. def _solve_sparse_cg(x, y, alpha, max_iter = none, tol = 1e-3, verbose = 0): n_samples, n_feat res = x.shape x1 = sp_linalg.aslinearoperator(x) coefs = np.empty((y.shape[1], n_feat res)) if n_feat res  n_samples: def create_mv(c rr_alpha): def _mv(x): ret rn x1.matvec(x1.rmatvec(x)) + c rr_alpha * x ret rn _mv else: def create_mv(c rr_alpha): def _mv(c rr_alpha): ret rn x1.rmatvec(x1.matvec(x)) + c rr_alpha * x ret rn _mv for i in range(y.shape[1]): y_col mn = y[:, i] mv = create_mv(alpha[i]) if n_feat res  n_samples: c = sp_linalg.linearoperator((n_samples, n_samples), matvec = mv, dtype = x.dtype) coef, info = sp_linalg.cg(c, y_col mn, tol = tol) coefs[i] = x1.rmatvec(coef) else: y_col mn = x1.rmatvec(y_col mn) c = sp_linalg.linearoperator((n_feat res, n_feat res), matvec = mv, dtype = x.dtype) coefs[i], info = sp_linalg.cg(c, y_col mn, maxiter = max_iter, tol = tol) if info  0: raise val eerror("failed with error code %d" % info) if max_iter is none and info  0 and verbose: warning.warn("sparse_cg did not coverge sfter %d iterations." % info) ret rn coefs _solve_sparse_cg: 这里将在理论意义上求逆矩阵的过程转换为线性系统的解. 所用到的线性算子仅仅是定义了对于矩阵的变换,matvec 向量乘积变换, rmatvec 共轭转置变换. 函数分n_feat res 与n_samples的大小进行了分类,可以证明在取逆的情况 下是相等的.(kernel ridge regression.pdf) 返回系数 def _solve_lsqr(x, y, alpha, max_iter = none, tol = 1e-3): n_samples, n_feat res = x.shape coefs = np.empty((y.shape[1], n_feat res)) n_iter = np.empty(y.shape[1], dtype = np.int32) sqrt_alpha = np.sqrt(alpha) for i in range(y.shape[1]): y_col mn = y[:,i] info = sp_linalg.lsqr(x, y_col mn, damp = sqrt_alpha[i], atol = tol, btol= tol, iter_lim = max_iter) coefs[i] = info[0] n_iter[i] = info[2] ret rn coefs, n_iter _solve_lsqr: 返回线性系统的最小二乘解. def _solve_cholesky(x, y, alpha): n_samples, n_feat res = x.shape n_targets = y.shape[1] a = safe_sparse_dot(x.t, x, dense_o tp t = tr e) xy = safe_sparse_dot(x.t, y, dense_o tp t = tr e) one_alpha = np.array_eq al(alpha, len(alpha) * [alpha[0) if one_alpha: a.flat[::n_feat res + 1]  alpha[0] ret rn linalg.solve(a, xy, sym_pos = tr e, overwrite_a = tr e) else: coefs = np.empty([n_targets, n_feat res]) for coef, target , c rr_alpha in zip(coefs, xy.t, alpha): a.flat[::n_feat res + 1] += c rr_alpha coef[:] = linalg.solve(a, target, sym_pos = tr e, overwrite_a = false).ravel a.flat[::n_feat res + 1] -= c rr_alpha ret rn coefs _solve_cholesky: 由于在linalg.solve中选择了sym_pos为tr e,故指定了矩阵为对称正定矩阵, 采用cholesky分解来解(分解为三角阵的内积) a.flat[::n_feat res + 1]实现了对对角元素的简写. def _solve_cholesky_kernel(k, y, alpha, sample_weight = none, copy = false): n_samples = k.shape[0] n_targets = y.shape[1] if copy: k = k.copy alpha = np.atleast_1d(alpha) one_alpha = (alpha == alpha[0]).all has_sw = isinstance(sample_weight, np.ndarray) or sample_weight not in [1.0, none] if has_sw: sw = np.sqrt(np.atleast_1d(sample_weight)) y = y * sw[:, np.newaxis] k *= np.o ter(sw, sw) if one_alpha: k.flat[::n_samples + 1] += alpha[0] try: d al_coef = linalg.solve(k, y, sym_pos = tr e, overwrite_a = false) except np.linalg.linalgerror: warning.warn("sing lar matrix in solving d al problem. sing " "least-sq ares sol tion instead.") d al_coef = linalg.lstsq(k, y)[0] k.flat[::n_samples + 1] -= alpha[0] if has_sw: d al_coef *= sw[:,np.newaxis] ret rn d al_coef else: d al_coefs = np.empty([n_targets, n_samples]) for d al_coef, target, c rrent_alpha in zip(d al_coefs, y.t, alpha): k.flat[::n_samples + 1] += c rrent_alpha d al_coef[:] = linalg.solve(k, target, sym_pos = tr e, overwrite_a = false).ravel k.flat[::n_samples + 1] -= c rrent_alpha if has_sw: d al_coefs *= sw[np.newaxis, :] ret rn d al_coefs.t _solve_cholesky_kernel: 该函数允许对样本赋予权重,唯一与_solve_cholesky_kernel的不同是其使用的核 矩阵需要给出. 还提供了当解等式linalg.solve失效时使用linalg.lstsq求最小二乘解的方案. def _solve_svd(x, y, alpha): u, s, vt = linalg.svd(x, f ll_matrices = false) idx = s  1e-15 s_nnz = s[idx][:,np.newaxis] uty = np.dot(u.t, y) d = np.zeros((s.size, alpha.size)) d[idx] = s_nnz  (s_nnz ** 2 + alpha) d_ut_y = d * uty ret rn np.dot(vt.t, d_ut_y).t _solve_svd: 这里仅仅是将上面_solve_cholesky_kernel的过程先将x进行奇异值分解, 将奇异值推导的岭回归结果求解出来. 这里以1e-15为阈值,砍掉了小的奇异值(与scipy.linalg.pinv求广义逆矩阵 特征值阈值相同) def ridge_regression(x, y, alpha, sample_weight = none, solver = a to , max_iter = none, tol = ie-3, verbose = 0, random_state = none, ret rn_n_iter = false, ret rn_intercept = false): if ret rn_intercept and sparse.issparse(x) and solver != sag : if solver != a to : warning.warn("in ridge, only sag solver can c rrently fit the " "intercept when x is sparse. solver has been a tomatically " "changed into sag .") solver = sag if solver == sag : x = check_array(x, accept_sparse = [ csr ], dtype = np.float64, order = c ) y = check_array(y, dtype = np.float64, ens re_2d = false, order = f ) else: x = check_array(x, accept_sparse = [ csr , csc , coo ], dtype = np.float64) y = check_array(y, dtype = n meric , ens re_2d = false) check_consistent_length(x, y) n_samples, n_feat res = x.shape if y.ndim  2: raise val eerror("target y has the wrong shape %s" % str(y,shape)) ravel = false if y.ndim == 1: y = y.reshpe(-1, 1) ravel = tr e n_samples_, n_targets = y.shape if n_samples != n_samples_: raise val eerror("n mber of samples in x and y does not correspond:" " %d != %d" % (n_samples, n_samples_)) has_sw = sample_weight is not none if solver == a to : if not sparse.issparse(x) or has_sw: solver = cholesky else: solver = sparse_cg elif solver == lsqr and not hasattr(sp_linalg, lsqr ): warning.warn("lsqr not avaliable on this machine, falling back to sparse_cg") solver = sparse_cg if has_sw: if np.atleast_1d(sample_weight).ndim  1: raise val eerror("sample weights m st be 1d array or scalar") if solver != sag : x, y = _rescale_data(x, y, sample_weight) alpha = np.asarray(alpha).ravel if alpha.size not in [1, n_targets]: raise val eerror("n mber of targets nad n mber of penalties " "do nt correspond: %d != %d" % (alpha.size, n_targets)) if alpha.size == 1 and n_targets  1: alpha = np.repeat(alpha, n_targets) if solver not in ( sparse_cg , cholesky , svd , lsqr , sag ): raise val eerror( solver %s not nderstood % solver) n_iter = none if solver == sparse_cg : coef = _solve_sparse_cg(x, y, alpha, max_iter, tol, verbose) elif solver == lsqr : coef, n_iter = _solve_lsqr(x, y, alpha, max_iter, tol) elif solver == cholesky : if n_feat res  n_samples: k = safe_sparse_dot(x, x.t, dense_o tp t = tr e) try: d al_coef = _solve_cholesky_kernel(k, y, alpha) coef = safe_sparse_dot(x.t, d al_coef, dense_o tp t = tr e).t except linalg.linalgerror: solver = svd else: try: coef = _solve_cholesky(x, y, alpha) except linalg.linalgerror: solver = svd elif solver == sag : max_sq ared_s m = row_norms(x, sq ared = tr e).max coef = np.empty([y.shape[1], n_feat res]) n_iter = np.empty(y.shape[1], dtype = np.int32) intercept = np.zeros((y.shape[1],)) for i, (alpha_i, target) in en merate(zip(alpha, y.t)): init = { coef : np.zeros((n_feat res + int(ret rn_intercept), 1))} coef_, n_iter_, _ = sag_solver(x, target.ravel, sample_weight, sq ared , alpha_i, max_iter, tol, verbose, random_state, false, max_sq ared_s m, init) if ret rn_intercept: coef[i] = coef_[:-1] intercept[i] = coef_[-1] else: coef[i] = coef_ n_iter[i] = n_iter_ if intercept.shape[0] == 1: intercept = intercept[0] coef = np.asarray(coef) if solver == svd : if sparse.issparse(x): raise typeerror( svd solver does not s pport sparse inp ts c rrently ) coef = _solve_svd(x, y, alpha) if ravel: coef = coef.ravel if ret rn_n_iter and ret rn_intercept: ret rn coef, n_iter, intercept elif ret rn_intercept: ret rn coef, intercept elif ret rn_n_iter: ret rn coef, n_iter else: ret rn coef ridge_regression: 这仅仅是对上面所提到的具体方法的统一接口. 岭回归当数据阵为sparse时仅能用随机梯度下降法"sag"求解. 当solver选项为 a to 时,对非稀疏矩阵或有加权的情况,采用cholesky 分解法解,否则使用sparse_cg n_feat res  n_samples 在cholesky分解下采用核方法(仅仅是不变的线性核) 从这里可以看到当矩阵为奇异时采用svd奇异值分解求解. svd不支持稀疏矩阵求解.