<jupyter_start><jupyter_text>一、逻辑回归（二分类）---  读取数据并可视化<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import scipy.optimize as opt
data = pd.read_csv('ex2data1.txt',names = ['exam 1 score','exam 2 score','admitted?'])
data.head()
df = data.copy()
col = df.shape[1]
x = df.iloc[:,0:col - 1]
y = df.iloc[:,col - 1 :col]
X = x.copy()
X.insert(0,'one',1)
x = np.array(x)
y = np.array(y)
X = np.array(X)
x_1 = x[:,0]
x_2 = x[:,1]
x,y,X,x_1,x_2,x_1,y.shape
"""将原数据分组"""
pos = []
neg = []
for i in range(x.shape[0]):#选出正类和负类的索引
    if y[i] == 1:
        pos.append(i)
    else:
        neg.append(i)
pos_x_1 = x_1[pos]#一维矩阵不能用于scatter中，要转换为一维数组或列表
pos_x_2 = x_2[pos]
neg_x_1 = x_1[neg]
neg_x_2 = x_2[neg]
fig,ax = plt.subplots()
ax.scatter(pos_x_1,pos_x_2,label = 'admitted',marker = 'o')
ax.scatter(neg_x_1,neg_x_2,label = 'not admitted',marker= 'x')
ax.legend(loc = 1)<jupyter_output><empty_output><jupyter_text>sigmoid函数<jupyter_code>def sigmoid(z):
    g = 1/(1+np.exp(-z))
    return g
n = np.arange(-10.,10.,0.1)
sigmoid(n)
fig = plt.figure()
plt.plot(n,sigmoid(n))
plt.show()<jupyter_output><empty_output><jupyter_text>代价函数<jupyter_code>theta = np.zeros(3)
def costfunction(theta, X,y):
    theta = theta.reshape(X.shape[1],1)
    cost_pos = -y * np.log(sigmoid(X @ theta))
    cost_neg = -(1-y) * np.log(1 - sigmoid(X @ theta))
    cost = 1/len(X)*(cost_pos + cost_neg)
    return np.sum(cost)
costfunction(theta,X,y)
y.shape,X.shape,theta.shape<jupyter_output><empty_output><jupyter_text>梯度下降<jupyter_code>"""一个梯度步长"""
def gradient(theta,X,y):
    theta = theta.reshape(X.shape[1],1)
    h = X @ theta
    erro = sigmoid(h) - y
    grad = (erro.T @ X)/len(X)
    return grad.ravel()
gradient(theta,X,y)
"""迭代10000次"""
costs = [costfunction(theta,X,y)]
for i in range(100000):
    theta = theta.reshape(X.shape[1],1)
    temp = theta - 0.01*gradient(theta, X,y).reshape(X.shape[1],1)
    theta = temp
    costs.append(costfunction(theta,X,y))
costs
fig  =plt.figure()
plt.plot(costs)
theta
##在alpha=0.01 迭代10000次后，代价来回跳动，依然很难收敛
"""用TNC优化并绘制决策界限"""
import scipy.optimize as opt
result = opt.minimize(fun=costfunction, x0=theta, args=(X,y), method='TNC')
result  ##参数类型难搞
theta = result.x[:,np.newaxis]
xx = np.arange(20,100,0.1)
theta = theta / theta[2]
yy = -theta[0] - theta[1] *xx
fig = plt.figure()
plt.plot(xx,yy)
plt.scatter(pos_x_1,pos_x_2,label = 'admitted',marker = 'o')
plt.scatter(neg_x_1,neg_x_2,label = 'not admitted',marker= 'x')
plt.show()
"""用sklearn中的逻辑回归"""
import sklearn.linear_model as lm
classfiser =lm.LogisticRegression(solver='liblinear', C=100)
y = y.ravel()
classfiser.fit(x,y)
x_min,x_max = min(X[:,1]) - 1.0, max(X[:,1]) + 1.0
y_min,y_max = min(X[:,2]) - 1.0, max(X[:,2]) + 1.0
x_values,y_values = np.meshgrid(np.arange(x_min,x_max,0.01),np.arange(y_min,y_max,0.1))
mesh_output = classfiser.predict(np.c_[x_values.ravel(),y_values.ravel()])
mesh_output = mesh_output.reshape(x_values.shape)
plt.figure()
plt.pcolormesh(x_values,y_values,mesh_output,cmap=plt.cm.gray)
plt.scatter(pos_x_1,pos_x_2,label = 'admitted',marker = 'o')
plt.scatter(neg_x_1,neg_x_2,label = 'not admitted',marker= 'x')
plt.show()<jupyter_output><empty_output><jupyter_text>二、逻辑回归正则化<jupyter_code>data2 = pd.read_csv("ex2data2.txt",names = ["text1","text2","quality"])
data2.head()
pos = data2[data2['quality'].isin([1])]
neg = data2[data2['quality'].isin([0])]

fig, ax = plt.subplots()
ax.scatter(pos['text1'], pos['text2'], s=50, c='b', marker='o', label='y = 1')
ax.scatter(neg['text1'], neg['text2'], s=50, c='r', marker='x', label='y = 0')
ax.legend()
ax.set_xlabel('text1')
ax.set_ylabel('text2')
plt.show()<jupyter_output><empty_output><jupyter_text>特征映射<jupyter_code>df2 = data2.copy()
x1 = df2["text1"]
x2 = df2["text2"]
x = np.array(df2.iloc[:,0:2])
def feature_map(x1,x2,power): 
    for i in np.arange(power+1):
        for j in np.arange(power+1):
            df2['F' + str(i) + str(j)] = np.power(x1, i) * np.power(x2, j)
    col = df2.shape[1]
    X = df2.iloc[:,3:col]
    X.insert(0,'one',1)
    X = np.array(X)
    y = np.array(df2.iloc[:,2:3])
    return df2,X,y
df2,X,y = feature_map(x1,x2,power = 5)
df2.head(),X,y<jupyter_output><empty_output><jupyter_text>正则化代价函数<jupyter_code>theta = np.zeros(X.shape[1])
X.shape,y.shape,theta.shape
def costfunctionreg(theta, X,y,c=1):
    theta = theta.reshape(X.shape[1],1)
    reg = c/len(X)*np.sum(theta[1:,:]**2)
    return costfunction(theta,X,y)+reg
costfunction(theta,X,y)<jupyter_output><empty_output><jupyter_text>正则化剃度下降<jupyter_code>"""正则化梯度步长"""
def gradientreg(theta,X,y,c=1):
    theta = theta.reshape((X.shape[1],1))
    reg = 1/len(X)*theta[1:,:]
    reg = np.vstack([np.zeros((1,1)),reg])
    return ((gradient(theta, X,y).reshape(X.shape[1],1) + reg)).ravel()
gradientreg(theta,X,y,c=1).shape
"""迭代11000次"""
def gradientreg_decent(theta,X,y,itera,c=1):
    costs = [costfunctionreg(theta,X,y,c=1)]
    for i in range(itera):
        theta = theta.reshape(X.shape[1],1)
        theta = theta - 0.005*gradientreg(theta,X,y,c=1).reshape(X.shape[1],1)
        costs.append(costfunctionreg(theta,X,y,c=1))
    return theta,costs
theta,costs = gradientreg_decent(theta, X, y,  itera = 11000, c=1)
plt.figure()
plt.plot(costs)
plt.show()
theta,costs[-1]<jupyter_output><empty_output><jupyter_text>拟合参数<jupyter_code>def opt_min(X,y):
    init_theta = np.zeros(X.shape[1])
    res = opt.minimize(fun=costfunctionreg,
                       x0=init_theta,
                       args=(X, y, 0.1),
                       method='TNC',
                       jac=gradientreg,
                       options={'maxiter': 10000})
    theta = res.x
    h = sigmoid(X @ theta[np.newaxis,:].T)
    h[h>=0.5] = 1
    h[h<0.5] = 0
    from sklearn.metrics import classification_report#这个包是评价报告
    return res,print(classification_report(y.ravel(), h.ravel()))
opt_min(X,y)<jupyter_output>precision    recall  f1-score   support

           0       0.90      0.77      0.83        60
           1       0.79      0.91      0.85        58

    accuracy                           0.84       118
   macro avg       0.85      0.84      0.84       118
weighted avg       0.85      0.84      0.84       118<jupyter_text>特征映射对分类器的影响<jupyter_code>df2,X,y = feature_map(x1,x2,1)
opt_min(X,y)
df2,X,y = feature_map(x1,x2,15)
opt_min(X,y)<jupyter_output>precision    recall  f1-score   support

           0       0.89      0.80      0.84        60
           1       0.81      0.90      0.85        58

    accuracy                           0.85       118
   macro avg       0.85      0.85      0.85       118
weighted avg       0.85      0.85      0.85       118<jupyter_text>画决策边界（不会）  逻辑回归以及准确率<jupyter_code>"""看看例子"""
theta = np.zeros((X.shape[1],1))
classifier = X @ theta
classifier.shape
classifier >= 0.5
np.where(classifier >= 0.5)
y_pred = np.zeros((X.shape[0],1))
y_pred[classifier >= 0.5] = 1
y_pred[classifier <= 0.5] = 0
y_pred
from sklearn.metrics import classification_report#这个包是评价报告
print(classification_report(y, y_pred))
"""逻辑回归，并返回准确率报告"""
def logistic_regression(X, y, itera):
    theta = np.zeros((X.shape[1], 1))
    final_theta,_ = gradientreg_decent(theta,X, y, itera, c=1 )
    classifier = sigmoid(X @ final_theta)#这就是训练好的分类器
    y_pred = np.zeros((X.shape[0], 1))
    y_pred[classifier >= 0.5] = 1
    y_pred[classifier <= 0.5] = 0 #二分类中置信度大于0.5的认为是正类，反之为负类
    from sklearn.metrics import classification_report#这个包是评价报告
    return print(classification_report(y, y_pred))#返回分类器的准确率   
logistic_regression(X, y, itera =1000)
logistic_regression(X, y, itera =10000)
logistic_regression(X, y, itera =12000)
logistic_regression(X, y, itera =20000)
logistic_regression(X, y, itera =40000)<jupyter_output>precision    recall  f1-score   support

           0       0.90      0.73      0.81        60
           1       0.77      0.91      0.83        58

    accuracy                           0.82       118
   macro avg       0.83      0.82      0.82       118
weighted avg       0.83      0.82      0.82       118<jupyter_text>随机参数上面不管是用高级优化法优化的模型，还是迭代了40000次的梯度下降的模型，在1的准确率都不高。我怀疑是初始参数的问题，下面尝试用随机的参数试试<jupyter_code>def randinit_theta(n, epsilon = 0.12):
    return np.random.uniform(-epsilon, epsilon, n)
df2,X,y = feature_map(x1,x2,10)
init_theta = randinit_theta(X.shape[1])
res = opt.minimize(fun=costfunctionreg,
                   x0=init_theta,
                   args=(X, y, 0.1),
                   method='TNC',
                   jac=gradientreg,
                   options={'maxiter': 10000})
theta = res.x
h = sigmoid(X @ theta[np.newaxis,:].T)
h[h>=0.5] = 1
h[h<0.5] = 0
from sklearn.metrics import classification_report#这个包是评价报告
res,print(classification_report(y.ravel(), h.ravel()))
init_theta = randinit_theta(X.shape[1])
final_theta,_ = gradientreg_decent(theta,X, y, itera = 30000, c=1 )
classifier = sigmoid(X @ final_theta)#这就是训练好的分类器
y_pred = np.zeros((X.shape[0], 1))
y_pred[classifier >= 0.5] = 1
y_pred[classifier <= 0.5] = 0 #二分类中置信度大于0.5的认为是正类，反之为负类
print(classification_report(y, y_pred))#返回分类器的准确率<jupyter_output>precision    recall  f1-score   support

           0       0.92      0.73      0.81        60
           1       0.77      0.93      0.84        58

    accuracy                           0.83       118
   macro avg       0.84      0.83      0.83       118
weighted avg       0.85      0.83      0.83       118
{'fraction_non_alphanumeric': 0.1677701836821871, 'fraction_numerical': 0.058842375053395984, 'mean_word_length': 3.2782092279579715, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '435700', 'license': 'MIT', 'n_tokens_mistral': 4409, 'n_tokens_neox': 4248, 'n_words': 631, 'path': 'code-homework/ML/ex2_Logistic Regression/ex2_logistic regressionV2.ipynb', 'repo_name': 'phww/Andrew.Ng-ML-Study', 'size': '9364'}