<jupyter_start><jupyter_text>数据科学基础 第四次作业<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from matplotlib.transforms import Bbox
from sklearn.datasets import make_classification
from sklearn.metrics import classification_report, zero_one_loss
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression, LinearRegression
% matplotlib inline<jupyter_output><empty_output><jupyter_text>实验一：决策树深度对模型性能的影响<jupyter_code>n_f = 30                    # 特征个数
inf_f = int(0.6 * n_f)   # 60%含信息特征
red_f = int(0.1 * n_f)   # 10%冗余特征(含信息特征的线性组合)
rep_f = int(0.1 * n_f)   # 10%重复特征(随机取自含信息特征和冗余特征)

# 创建分类数据集
X,y = make_classification(
    n_samples=1000,        # 实例个数
    flip_y=0.2,           # 随机抽20%实例改变类别,产生一些噪声
    n_features=n_f,
    n_informative=inf_f,
    n_redundant=red_f,
    n_repeated=rep_f,
    random_state=7)
# 划分为训练集,验证集,测试集
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.5,random_state=9)
# 验证模型深度的影响
single_train = []
single_test = []
for i in range(1,21):
    # 单决策树模型
    model = DecisionTreeClassifier(max_depth=i,min_samples_leaf=1)
    model.fit(X_train,y_train)
    # 单模型在验证集上预测
    y_pred = model.predict(X_train)
    single_train.append(1-zero_one_loss(y_train,y_pred))
    y_pred = model.predict(X_test)
    single_test.append(1-zero_one_loss(y_test,y_pred))
plt.figure(figsize=(10,4))
x_axis=range(1,21)
plt.plot(x_axis,single_train,marker='.',markersize=10)
plt.plot(x_axis,single_test,marker='.',markersize=10)
plt.plot([5,5],[0.6,1.05],c='r',linestyle='--')
plt.ylim([0.6,1.05])
plt.xticks(x_axis)
plt.xlabel('Depth')
plt.ylabel('Accuracy')
plt.legend(['train','test'])
plt.savefig('./figure/depth.pdf')
plt.show()<jupyter_output><empty_output><jupyter_text>实验二：单独模型及组合模型性能对比<jupyter_code># 三种模型对比

# 单决策树模型
model = DecisionTreeClassifier(max_depth=5,min_samples_leaf=1)
model.fit(X_train,y_train)

# boosting集成
boosting = AdaBoostClassifier(
    DecisionTreeClassifier(max_depth=5,min_samples_leaf=1), 
    n_estimators=10,
    algorithm="SAMME",   
    random_state=9)
boosting.fit(X_train,y_train)

# bagging集成
bagging = BaggingClassifier(
    DecisionTreeClassifier(max_depth=5,min_samples_leaf=1), 
    n_estimators=10,
    bootstrap=True,
    max_samples=1.0,
    bootstrap_features=True,
    max_features=0.7,
    random_state=9)
bagging.fit(X_train,y_train)

# 单模型在验证集上预测
y_pred = model.predict(X_train)
print classification_report(y_train,y_pred)
print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_train,y_pred)*100),"%"
y_pred = model.predict(X_test)
print classification_report(y_test,y_pred)
print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_test,y_pred)*100),"%"

# 集成模型在验证集上预测 boosting
y_pred = boosting.predict(X_train)
print classification_report(y_train,y_pred)
print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_train,y_pred)*100),"%"
y_pred = boosting.predict(X_test)
print classification_report(y_test,y_pred)
print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_test,y_pred)*100),"%"

# 集成模型在验证集上预测 bagging
y_pred = bagging.predict(X_train)
print classification_report(y_train,y_pred)
print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_train,y_pred)*100),"%"
y_pred = bagging.predict(X_test)
print classification_report(y_test,y_pred)
print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_test,y_pred)*100),"%"<jupyter_output>precision    recall  f1-score   support

          0       0.81      0.74      0.77      2475
          1       0.76      0.83      0.79      2525

avg / total       0.78      0.78      0.78      5000

Fraction of misclassification = 21.74 %
             precision    recall  f1-score   support

          0       0.79      0.70      0.74      2521
          1       0.73      0.81      0.77      2479

avg / total       0.76      0.76      0.76      5000

Fraction of misclassification = 24.36 %
             precision    recall  f1-score   support

          0       0.90      0.84      0.87      2475
          1       0.86      0.91      0.88      2525

avg / total       0.88      0.88      0.88      5000

Fraction of misclassification = 12.18 %
             precision    recall  f1-score   support

          0       0.83      0.77      0.80      2521
          1       0.78      0.85      0.81      2479

avg / total       0.81      0.81      0.80      5000

Fraction of misclass[...]<jupyter_text>实验三：分类器数量对组合模型性能的影响<jupyter_code>boosting_acc = []
bagging_acc = []

for i in range(10,301,10):
    print(i)
    # boosting集成
    boosting = AdaBoostClassifier(
        DecisionTreeClassifier(max_depth=5,min_samples_leaf=1), 
        n_estimators=i,
        algorithm="SAMME",
        random_state=9)
    boosting.fit(X_train,y_train)
    
    # bagging集成
    bagging = BaggingClassifier(
        DecisionTreeClassifier(max_depth=5,min_samples_leaf=1), 
        n_estimators=i,
        bootstrap=True,
        max_samples=1.0,
        bootstrap_features=True,
        max_features=0.7,
        random_state=9)
    bagging.fit(X_train,y_train)
    
    y_pred = boosting.predict(X_test)
    print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_test,y_pred)*100),"%"
    boosting_acc.append(1-zero_one_loss(y_test,y_pred))
    y_pred = bagging.predict(X_test)
    print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_test,y_pred)*100),"%"
    bagging_acc.append(1-zero_one_loss(y_test,y_pred))
plt.figure(figsize=(10,4))
x_axis = range(10,301,10)
plt.plot(x_axis,boosting_acc,marker='.',markersize=10)
plt.plot(x_axis,bagging_acc,marker='.',markersize=10)
plt.ylim([0.8,0.86])
plt.xlabel('#Estimator')
plt.ylabel('Accuracy')
plt.legend(['Boosting','Bagging'])
plt.savefig('./figure/nEstimator.pdf')
plt.show()<jupyter_output><empty_output><jupyter_text>实验四：数据集噪声率对组合模型性能的影响<jupyter_code>single_acc2 = []
boosting_acc2 = []
bagging_acc2 = []
for i in range(51):
    print(i)
    # 创建分类数据集
    X,y = make_classification(
        n_samples=1000,
        flip_y=0.01*i,
        n_features=n_f,
        n_informative=inf_f,
        n_redundant=red_f,
        n_repeated=rep_f,
        random_state=7)
    # 划分为训练集,验证集,测试集
    X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.5,random_state=9)
    
    # 单模型
    model = DecisionTreeClassifier(max_depth=5,min_samples_leaf=1)
    model.fit(X_train,y_train)
    
    # boosting集成
    boosting = AdaBoostClassifier(
        DecisionTreeClassifier(max_depth=5,min_samples_leaf=1), 
        n_estimators=50,
        algorithm="SAMME",
        random_state=9)
    boosting.fit(X_train,y_train)
    
    # bagging集成
    bagging = BaggingClassifier(
        DecisionTreeClassifier(max_depth=5,min_samples_leaf=1), 
        n_estimators=50,
        bootstrap=True,
        max_samples=1.0,
        bootstrap_features=True,
        max_features=0.7,
        random_state=9)
    bagging.fit(X_train,y_train)
    
    y_pred = model.predict(X_test)
    print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_test,y_pred)*100),"%"
    single_acc2.append(1-zero_one_loss(y_test,y_pred))
    y_pred = boosting.predict(X_test)
    print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_test,y_pred)*100),"%"
    boosting_acc2.append(1-zero_one_loss(y_test,y_pred))
    y_pred = bagging.predict(X_test)
    print "Fraction of misclassification = %0.2f" % (zero_one_loss(y_test,y_pred)*100),"%"
    bagging_acc2.append(1-zero_one_loss(y_test,y_pred))
# flip比例的影响
# 比较准确率的均值和方差，画图
boo_acc2 = np.array(boosting_acc2)
bag_acc2 = np.array(bagging_acc2)
sin_acc2 = np.array(single_acc2)
print(boo_acc2.mean(),boo_acc2.std())
print(bag_acc2.mean(),bag_acc2.std())
print(sin_acc2.mean(),sin_acc2.std())

plt.figure(figsize=(10,5))
x_axis = np.arange(0,51)*0.01
plt.plot(x_axis,boosting_acc2,marker='.',markersize=10)
plt.plot(x_axis,bagging_acc2,marker='.',markersize=10)
# plt.ylim([0.8,0.86])
plt.xlabel('Flip%')
plt.ylabel('Accuracy')
plt.legend(['Boosting','Bagging'])
plt.savefig('./figure/flip.pdf')
plt.show()<jupyter_output>(0.76031372549019605, 0.07923395246594496)
(0.76737254901960772, 0.054794432515347477)
(0.67227450980392145, 0.050077617571350701)<jupyter_text>实验五：模型性能综合对比<jupyter_code># 创建分类数据集
X,y = make_classification(
    n_samples=1000, 
    flip_y=0.2,
    n_features=n_f,
    n_informative=inf_f,
    n_redundant=red_f,
    n_repeated=rep_f,
    random_state=7)
# 划分为训练集,验证集,测试集
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.5,random_state=9)
# 决策树

# 单模型
model = DecisionTreeClassifier(max_depth=5,min_samples_leaf=1)
model.fit(X_train,y_train)

# boosting集成
boosting = AdaBoostClassifier(
    DecisionTreeClassifier(max_depth=5,min_samples_leaf=1), 
    n_estimators=50,
    algorithm="SAMME",
    random_state=9)
boosting.fit(X_train,y_train)

# bagging集成
bagging = BaggingClassifier(
    DecisionTreeClassifier(max_depth=5,min_samples_leaf=1), 
    n_estimators=50, 
    bootstrap=True,
    max_samples=1.0,    
    bootstrap_features=True,
    max_features=0.7,      
    random_state=9)
bagging.fit(X_train,y_train)

# 单模型在验证集上预测
y_pred = model.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))

# 集成模型在验证集上预测 boosting
y_pred = boosting.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))

# 集成模型在验证集上预测 bagging
y_pred = bagging.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))
# KNN

# 单模型
model = KNeighborsClassifier()
model.fit(X_train,y_train)

# bagging集成
bagging = BaggingClassifier(
    KNeighborsClassifier(), 
    n_estimators=50,  
    bootstrap=True,
    max_samples=1.0, 
    bootstrap_features=True,
    max_features=0.7, 
    random_state=9)
bagging.fit(X_train,y_train)

# 单模型在验证集上预测
y_pred = model.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))

# 集成模型在验证集上预测 bagging
y_pred = bagging.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))
# SVM

# 单模型
model = SVC()
model.fit(X_train,y_train)

# boosting集成
boosting = AdaBoostClassifier(
    SVC(), 
    n_estimators=50,
    algorithm="SAMME",
    random_state=9)
boosting.fit(X_train,y_train)

# bagging集成
bagging = BaggingClassifier(
    SVC(), 
    n_estimators=50,
    bootstrap=True,
    max_samples=1.0,
    bootstrap_features=True,
    max_features=0.7,
    random_state=9)
bagging.fit(X_train,y_train)

# 单模型在验证集上预测
y_pred = model.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))

# 集成模型在验证集上预测 boosting
y_pred = boosting.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))

# 集成模型在验证集上预测 bagging
y_pred = bagging.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))
# Logistic回归

# 单模型
model = LogisticRegression()
model.fit(X_train,y_train)

# boosting集成
boosting = AdaBoostClassifier(
    LogisticRegression(), 
    n_estimators=50,
    algorithm="SAMME",
    random_state=9)
boosting.fit(X_train,y_train)

# bagging集成
bagging = BaggingClassifier(
    LogisticRegression(), 
    n_estimators=50,
    bootstrap=True,
    max_samples=1.0,
    bootstrap_features=True,
    max_features=0.7,
    random_state=9)
bagging.fit(X_train,y_train)

# 单模型在验证集上预测
y_pred = model.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))

# 集成模型在验证集上预测 boosting
y_pred = boosting.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))

# 集成模型在验证集上预测 bagging
y_pred = bagging.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))
# 随机森林

model = RandomForestClassifier(n_estimators=50)
model.fit(X_train,y_train)

# 在验证集上预测
y_pred = model.predict(X_test)
print classification_report(y_test,y_pred)
print "Accuracy = %0.4f" % (1-zero_one_loss(y_test,y_pred))
plt.figure(figsize=(6,6))

# KNN
plt.scatter(0.808,0.81,marker='.',s=400,c='r')
plt.scatter(0.852,0.85,marker='x',s=100,c='r')
# 决策树
plt.scatter(0.722,0.72,marker='.',s=400,c='b')
plt.scatter(0.794,0.79,marker='*',s=200,c='b')
plt.scatter(0.814,0.81,marker='x',s=100,c='b')
# SVM
plt.scatter(0.752,0.74,marker='.',s=400,c='g')
plt.scatter(0.554,0.4,marker='*',s=200,c='g')
plt.scatter(0.79,0.78,marker='x',s=100,c='g')
# Logistic回归
plt.scatter(0.734,0.73,marker='.',s=400,c='y')
plt.scatter(0.76,0.76,marker='*',s=200,c='y')
plt.scatter(0.788,0.79,marker='x',s=100,c='y')
# 随机森林
plt.scatter(0.82,0.82,marker='.',s=400,c='m')

plt.legend(['KNN','KNN(Bagging)','DecisionTree','DecisionTree(Boosting)','DecisionTree(Bagging)','SVM','SVM(Boosting)','KNN(Bagging)','LogosticRegression','LogosticRegression(Boosting)','LogosticRegression(Bagging)','RandomForest'],
          loc='center left',bbox_to_anchor=(1.3, 0.5),fontsize=18)
plt.xlim([0.7,0.86])
plt.ylim([0.7,0.86])
plt.xlabel('Accuracy',fontsize=18)
plt.ylabel('F1-Score',fontsize=18)
plt.xticks([0.70,0.75,0.80,0.85],[0.70,0.75,0.80,0.85],fontsize=18)
plt.yticks([0.75,0.80,0.85],[0.75,0.80,0.85],fontsize=18)
plt.savefig('./figure/models.pdf',bbox_inches=Bbox([[-0.5,0],[12,6]]))
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13901247173802056, 'fraction_numerical': 0.05871198307927941, 'mean_word_length': 4.610474631751227, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '212191', 'license': 'MIT', 'n_tokens_mistral': 6371, 'n_tokens_neox': 6068, 'n_words': 694, 'path': 'homework_4/code/.ipynb_checkpoints/hw4-checkpoint.ipynb', 'repo_name': 'chenxi-wang/CS245-Introduction-To-Data-Science', 'size': '13711'}