<commit_before>import numpy as np
import matplotlib.pylab as plt
from sklearn.datasets import make_blobs
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import KFold
from sklearn.utils import resample
from sklearn.metrics import zero_one_loss

def model_selection(clf, parameter, parameter_range): 
    fig, ax = plt.subplots(1, 1)
    
    ## generate dataset
    n_samples, n_centers = 10000, 10
    X, y = make_blobs(n_samples = n_samples, n_features = 2, centers = n_centers, random_state = 100)
    # y = np.take([True, False], (y < 5))
    
    misclassify_rate_k_fold = []
    for k in parameter_range:
        para = {parameter: k}
        clf.set_params(**para)
        scores = []
        kfold = KFold(n = X.shape[0], n_folds = 10)
        for train, test in kfold:
            X_train, y_train = X[train], y[train]
            X_test, y_test = X[test], y[test]
            clf.fit(X_train, y_train)
            scores.append(zero_one_loss(y_test, clf.predict(X_test)))
        misclassify_rate_k_fold.append([np.mean(scores), np.std(scores)])
    
    misclassify_rate = np.array(misclassify_rate_k_fold, dtype = np.float64)
    ax.plot(parameter_range, misclassify_rate[:, 0], label = '10-fold validation mean')
    ax.plot(parameter_range, misclassify_rate[:, 1], label = '10-fold validation std')

    misclassify_rate_bt = []
    ## bootstraping
    for k in parameter_range:
        para = {parameter: k}
        clf.set_params(**para)
        scores = []
        for i in xrange(0, 100):
            X_train, y_train = resample(X, y)
            clf.fit(X_train, y_train)
            scores.append(zero_one_loss(y, clf.predict(X)))
        
        misclassify_rate_bt.append([np.mean(scores), np.std(scores)])
    
    misclassify_rate = np.array(misclassify_rate_bt, dtype = np.float64)
    ax.plot(parameter_range, misclassify_rate[:, 0], label = '100 boostraping mean')
    ax.plot(parameter_range, misclassify_rate[:, 1], label = '100 boostraping std')
    
    ax.set_ylim(0.0)
    ax.set_xlabel('k')
    ax.set_ylabel('Misclassification rate')
    ax.legend(loc = 'best', fontsize = 'medium')
    
    plt.show()

if __name__ == '__main__':
    model_selection(KNeighborsClassifier(), 'n_neighbors', xrange(1, 50))
    model_selection(DecisionTreeClassifier(), 'max_depth', xrange(3, 10)) # max_leaf_nodes<commit_msg>Add bias and variance.<commit_after>import numpy as np
import matplotlib.pylab as plt
from sklearn.datasets import make_blobs
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import KFold
from sklearn.utils import resample
from sklearn.metrics import zero_one_loss
from numpy import dtype

def model_selection(clf, parameter, parameter_range): 
    fig, ax = plt.subplots(1, 1)
    
    ## generate dataset
    n_samples, n_centers = 10000, 10
    X, y = make_blobs(n_samples = n_samples, n_features = 2, centers = n_centers, random_state = 100)
    y = np.take([True, False], (y < n_centers / 2))
    
    misclassify_rate_k_fold = []
    for k in parameter_range:
        para = {parameter: k}
        clf.set_params(**para)
        scores = []
        kfold = KFold(n = X.shape[0], n_folds = 10)
        for train, test in kfold:
            X_train, y_train = X[train], y[train]
            X_test, y_test = X[test], y[test]
            clf.fit(X_train, y_train)
            scores.append(zero_one_loss(y_test, clf.predict(X_test)))
        misclassify_rate_k_fold.append([np.mean(scores), np.std(scores)])
    
    misclassify_rate = np.array(misclassify_rate_k_fold, dtype = np.float64)
    ax.plot(parameter_range, misclassify_rate[:, 0], label = '10-fold validation mean')
    ax.plot(parameter_range, misclassify_rate[:, 1], label = '10-fold validation std')

    misclassify_rate_bt, biases, variences = [], [], []
    ## bootstraping
    for k in parameter_range:
        para = {parameter: k}
        clf.set_params(**para)
        scores = []
        n_replicas = 10
        counts = np.zeros(X.shape[0], dtype = np.int64)
        sum_preds = np.zeros(X.shape[0], dtype = np.float64)
        for it in xrange(n_replicas):
            # generate train sets and test sets
            train_indices = np.random.randint(X.shape[0], size = X.shape[0])
            
            in_train = np.unique(train_indices)
            mask = np.ones(X.shape[0], dtype = np.bool)
            mask[in_train] = False
            test_indices = np.arange(X.shape[0])[mask]
            
            clf.fit(X[train_indices], y[train_indices])
            
            scores.append(zero_one_loss(y[test_indices], clf.predict(X[test_indices])))
            
            preds = clf.predict(X)
            for index in test_indices:
                counts[index] += 1
                sum_preds[index] += preds[index]
        
        mask = (counts == 0)
        counts[mask] = 1 # avoid divide by 0
        main_preds = np.round(sum_preds / counts)
        
        main_preds[mask] = y[mask] # not test once
        
        sample_bias = np.abs(y - main_preds)
        sample_var = np.abs(sum_preds / counts - main_preds)
        
        bias = np.mean(sample_bias)
        biases.append(bias)
        u_var = np.sum(sample_var[sample_bias == 0]) / float(X.shape[0])
        b_var = np.sum(sample_var[sample_bias != 0]) / float(X.shape[0])
        var = u_var - b_var
        variences.append(var)
        
        misclassify_rate_bt.append([np.mean(scores), np.std(scores)])
    
    misclassify_rate = np.array(misclassify_rate_bt, dtype = np.float64)
    ax.plot(parameter_range, misclassify_rate[:, 0], label = '100 boostraping mean')
    ax.plot(parameter_range, misclassify_rate[:, 1], label = '100 boostraping std')
    ax.plot(parameter_range, biases, label = '100 boostraping bias')
    ax.plot(parameter_range, variences, label = '100 boostraping variance')
    
    ax.set_ylim(0.0)
    ax.set_xlabel('k')
    ax.set_ylabel('Misclassification rate')
    ax.legend(loc = 'best', fontsize = 'medium')
    
    plt.show()

if __name__ == '__main__':
    model_selection(KNeighborsClassifier(), 'n_neighbors', xrange(1, 50))
    model_selection(DecisionTreeClassifier(), 'max_depth', xrange(3, 10)) # max_leaf_nodes
{'fraction_non_alphanumeric': 0.11801930074355324, 'fraction_numerical': 0.0183515266571745, 'mean_word_length': 2.8643031784841075, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2051712', 'n_tokens_mistral': 2257, 'n_tokens_neox': 2152, 'n_words': 451}