<jupyter_start><jupyter_text>K Cross Validation<jupyter_code>kfold = StratifiedKFold(y = y_train, n_folds = 10, random_state = 0)
scores = cross_val_score(estimator = pipeline, X = X_train, y = y_train, cv = 10, n_jobs = -1)
# n_jobs = -1: use all CPU cores for parallel computing

print("CV accuracy scores: ", scores)
print("Mean CV accuracy: %.3f, std: %.3f" % (np.mean(scores), np.std(scores)))
plt.plot(scores)
plt.ylim(0.9, 1.01)
plt.xlabel("Iteration")
plt.ylabel("Accuracy Score")
train_sizes, train_scores, test_scores = learning_curve(estimator = pipeline, X = X_train, y = y_train,
                                                        train_sizes = np.linspace(0.1, 1.0, 10),
                                                        cv = 10, 
                                                        n_jobs = -1)

# learning_curve internally uses stratified_kfold
train_mean = np.mean(train_scores, axis = 1)
test_mean = np.mean(test_scores, axis = 1)
train_std = np.std(train_scores, axis = 1)
test_std = np.std(test_scores, axis = 1)
plt.figure(figsize = (15, 10))
plt.plot(train_sizes, train_mean, color = "b", marker = "o", markersize = 5, label = "training accuracy scores")
plt.fill_between(train_sizes, train_mean + train_std, train_mean - train_std, alpha = 0.4)
plt.plot(train_sizes, test_mean, color = "g", ls = "--", marker = "s", markersize = 5, label = "validation accurancy")
plt.fill_between(train_sizes, test_mean + test_std, test_mean - test_std, alpha = 0.15, color = "g")


plt.legend(loc = "lower right")
plt.xlabel("Number of training samples")
plt.ylabel("Accuracy scores")<jupyter_output><empty_output><jupyter_text>Validation Curve<jupyter_code>np.set_printoptions(precision = 4, suppress= True)
param_range = 10.0 ** np.arange(-3, 4)
print("param_range", param_range)
train_scores, test_scores = validation_curve(estimator = pipeline, 
                                             X = X_train, 
                                             y = y_train, 
                                             param_name = "clf__C", 
                                             param_range = param_range,
                                             cv = 10)

# validation_curve internally uses stratified_kfold
train_mean = np.mean(train_scores, axis = 1)
test_mean = np.mean(test_scores, axis = 1)
train_std = np.std(train_scores, axis = 1)
test_std = np.std(test_scores, axis = 1)

plt.figure(figsize = (15, 5))
plt.plot(param_range, train_mean, color = "b", marker = "o", markersize = 5, label = "training accuracy scores")
plt.fill_between(param_range, train_mean + train_std, train_mean - train_std, alpha = 0.4)
plt.plot(param_range, test_mean, color = "g", ls = "--", marker = "s", markersize = 5, label = "validation accurancy")
plt.fill_between(param_range, test_mean + test_std, test_mean - test_std, alpha = 0.15, color = "g")


plt.legend(loc = "lower right")
plt.xlabel("Complexity Parameter - C")
plt.ylabel("Accuracy scores")
plt.xscale("log")
plt.ylim(0.8, 1.02)<jupyter_output><empty_output><jupyter_text>Hyper Parameter Tuning using Grid Search<jupyter_code>pipe_svc = Pipeline([
        ("scl", StandardScaler()),
        ("clf", SVC(random_state = 0))
    ])

param_range = 10.0 ** np.arange(-4, 4)
param_range
param_grid = [
    {"clf__C": param_range, "clf__kernel": ["linear"]},
    {"clf__C": param_range, "clf__gamma": param_range, "clf__kernel": ["rbf"]}
]
gs = GridSearchCV(estimator = pipe_svc, 
                 param_grid = param_grid,
                 scoring = "accuracy",
                 cv = 10, 
                 n_jobs = -1)
gs.fit(X_train, y_train)
print("Best params: %s Best score: %.4f" % (gs.best_params_, gs.best_score_))
# Applying Grid Search on Decision Tree
tree = DecisionTreeClassifier(random_state = 0)
param_grid = [
    {"max_depth": [1, 2, 3, 4, 5, 6, 7, None]}
]

gs_tree = GridSearchCV(estimator = tree, param_grid = param_grid, scoring = "accuracy", cv = 5)
gs_tree.fit(X_train, y_train)
print("Best param: %s, best score: %s" % (gs_tree.best_params_, gs_tree.best_score_))<jupyter_output>Best param: {'max_depth': 3}, best score: 0.8790322580645161
{'fraction_non_alphanumeric': 0.13452697798209534, 'fraction_numerical': 0.024679409629808857, 'mean_word_length': 3.037109375, 'pattern_counts': {'":': 6, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '14876', 'license': 'Apache-2.0', 'n_tokens_mistral': 1527, 'n_tokens_neox': 1380, 'n_words': 326, 'path': 'Scikit - 15 Learning Curve and Tuning .ipynb', 'repo_name': 'rajeevjnnce/machine-learning', 'size': '4133'}