<jupyter_start><jupyter_text>有过拟合的趋势，进行交叉检验将原始数据集划分为k个子集，用其中一个子集作为测试集剩下的所有子集用作训练集。这个过程将重复k次，每一个子集都有一次机会作为测试集<jupyter_code># 10-fold cross-validation 原理说明
import numpy as np
from sklearn.cross_validation import StratifiedKFold

def plot_cv(cv, n_samples):
    masks = []
    for train, test in cv:
        mask = np.zeros(n_samples, dtype=bool)
        mask[test] = 1
        masks.append(mask)
        
    plt.figure(figsize=(15, 15))
    plt.imshow(masks, interpolation='none')
    plt.ylabel('Fold')
    plt.xlabel('Row #')
    
plot_cv(StratifiedKFold(all_classes, n_folds=10), len(all_classes))
from sklearn.cross_validation import cross_val_score

decision_tree_classifier = DecisionTreeClassifier()

# cross_val_score(cv_scores)返回了一个list，我们可以对它进行可视化，来查看分类器的表现
cv_scores = cross_val_score(decision_tree_classifier, all_inputs, all_classes, cv=10)
sns.distplot(cv_scores)
plt.title('Average score: {}'.format(np.mean(cv_scores)))
plt.show()
# 调整超参数 max_depth，可以看到准确率下降严重
decision_tree_classifier = DecisionTreeClassifier(max_depth=1)

cv_scores = cross_val_score(decision_tree_classifier, all_inputs, all_classes, cv=10)
sns.distplot(cv_scores, kde=False)
plt.title('Average score: {}'.format(np.mean(cv_scores)))
plt.show()
# 探索 max_depth 和 max_features 进行调参
from sklearn.grid_search import GridSearchCV

decision_tree_classifier = DecisionTreeClassifier()

parameter_grid = {'max_depth': [1, 2, 3, 4, 5],
                  'max_features': [1, 2, 3, 4]}

cross_validation = StratifiedKFold(all_classes, n_folds=10)

grid_search = GridSearchCV(decision_tree_classifier,
                           param_grid=parameter_grid,
                           cv=cross_validation)

grid_search.fit(all_inputs, all_classes)
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
# 可视化调参过程

grid_visualization = []

for grid_pair in grid_search.grid_scores_:
    grid_visualization.append(grid_pair.mean_validation_score)
    
grid_visualization = np.array(grid_visualization)
grid_visualization.shape = (5, 4)
sns.heatmap(grid_visualization, cmap='Blues')
plt.xticks(np.arange(4) + 0.5, grid_search.param_grid['max_features'])
plt.yticks(np.arange(5) + 0.5, grid_search.param_grid['max_depth'][::-1])
plt.xlabel('max_features')
plt.ylabel('max_depth')
plt.show()
# 尝试调整更多的参数

decision_tree_classifier = DecisionTreeClassifier()

parameter_grid = {
    'criterion': ['gini', 'entropy'],
    'splitter': ['best', 'random'],
    'max_depth': [1, 2, 3, 4, 5],
    'max_features': [1, 2, 3, 4]
}

cross_validation = StratifiedKFold(all_classes, n_folds=10)

grid_search = GridSearchCV(decision_tree_classifier,
                          param_grid=parameter_grid,
                          cv=cross_validation)

grid_search.fit(all_inputs, all_classes)
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
decision_tree_classifier = grid_search.best_estimator_
decision_tree_classifier
dt_scores = cross_val_score(decision_tree_classifier, all_inputs, all_classes, cv=10)

sns.boxplot(dt_scores)
sns.stripplot(dt_scores, jitter=True, color='white')
# 引入一个新的分类器随机森林，和之前的决策数做对比

from sklearn.ensemble import RandomForestClassifier

random_forest_classifier = RandomForestClassifier()

parameter_grid = {
    'n_estimators': [5, 10, 25, 50],
    'criterion': ['gini', 'entropy'],
    'max_features': [1, 2, 3, 4],
    'warm_start': [True, False]
}

cross_validation = StratifiedKFold(all_classes, n_folds=10)

grid_search = GridSearchCV(random_forest_classifier,
                           param_grid=parameter_grid,
                           cv=cross_validation)

grid_search.fit(all_inputs, all_classes)
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))

grid_search.best_estimator_
# 比较两个分类器的表现
random_forest_classifier = grid_search.best_estimator_

rf_df = pd.DataFrame({
    'accuracy': cross_val_score(random_forest_classifier, all_inputs, all_classes, cv=10),
    'classifier': ['Random Forest'] * 10
})

dt_df = pd.DataFrame({
    'accuracy': cross_val_score(decision_tree_classifier, all_inputs, all_classes, cv=10),
    'classifier': ['Decision Tree'] * 10
})
both_df = rf_df.append(dt_df)

sns.boxplot(x='classifier', y='accuracy', data=both_df)
sns.stripplot(x='classifier', y='accuracy', data=both_df, jitter=True, color='white')
plt.show()<jupyter_output><empty_output><jupyter_text>两个分类器在这个数据集上表现比较相似。随机森林模型在数据集有多个特征时表现更好。  过程复用<jupyter_code># 将之前的步骤整理成一个可以复刻的 pipeline

%matplotlib inline
import pandas as pd
import seaborn as sb
from sklearn.ensemble import RandomForestClassifier
from sklearn.cross_validation import train_test_split, cross_val_score

iris_data = pd.read_csv('http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data', names = ['sepal_length_cm', 'sepal_width_cm', 'petal_length_cm', 'petal_width_cm', 'class'])

# input
all_inputs = iris_data[['sepal_length_cm', 'sepal_width_cm',
                             'petal_length_cm', 'petal_width_cm']].values
all_classes = iris_data['class'].values

# 分类器 & Grid Search
random_forest_classifier = RandomForestClassifier(bootstrap=True, class_weight=None,
                                                 criterion='gini', max_depth=None,
                                                 max_features=3, max_leaf_nodes=None,
                                                 min_samples_leaf=1, min_samples_split=2,
                                                 min_weight_fraction_leaf=0.0, n_estimators=5,
                                                 n_jobs=1, oob_score=False,
                                                 random_state=None, verbose=0,
                                                 warm_start=True)

# cross-validation scores
rf_classifier_scores = cross_val_score(random_forest_classifier, all_inputs, all_classes, cv=10)
sb.boxplot(rf_classifier_scores)
sb.stripplot(rf_classifier_scores, jitter=True, color='white')

# 展示其中分类器预测结果的一部分
(training_inputs,
 testing_inputs,
 training_classes,
 testing_classes) = train_test_split(all_inputs, all_classes, train_size=0.75)

random_forest_classifier.fit(training_inputs, training_classes)

for input_features, prediction, actual in zip(testing_inputs[:10],
                                              random_forest_classifier.predict(testing_inputs[:10]),
                                              testing_classes[:10]):
    print('{}\t-->\t{}\t(Actual: {})'.format(input_features, prediction, actual))<jupyter_output>[6.4 3.1 5.5 1.8]	-->	Iris-virginica	(Actual: Iris-virginica)
[4.9 2.5 4.5 1.7]	-->	Iris-versicolor	(Actual: Iris-virginica)
[6.2 2.9 4.3 1.3]	-->	Iris-versicolor	(Actual: Iris-versicolor)
[6.3 2.3 4.4 1.3]	-->	Iris-versicolor	(Actual: Iris-versicolor)
[5.1 3.5 1.4 0.3]	-->	Iris-setosa	(Actual: Iris-setosa)
[5.  3.2 1.2 0.2]	-->	Iris-setosa	(Actual: Iris-setosa)
[5.8 2.7 4.1 1. ]	-->	Iris-versicolor	(Actual: Iris-versicolor)
[7.4 2.8 6.1 1.9]	-->	Iris-virginica	(Actual: Iris-virginica)
[7.2 3.2 6.  1.8]	-->	Iris-virginica	(Actual: Iris-virginica)
[7.1 3.  5.9 2.1]	-->	Iris-virginica	(Actual: Iris-virginica)
{'fraction_non_alphanumeric': 0.12350652959155321, 'fraction_numerical': 0.022645179216449013, 'mean_word_length': 3.9106412005457027, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '610084', 'license': 'MIT', 'n_tokens_mistral': 2829, 'n_tokens_neox': 2728, 'n_words': 412, 'path': 'Iris Classification.ipynb', 'repo_name': 'bighuang624/Iris-Classification', 'size': '7198'}