<jupyter_start><jupyter_text>Load Pickled Labels<jupyter_code>directory = 'generated_data'
if not os.path.exists(directory):
    os.makedirs(directory)

train_labels = np.load('{0}/train_labels.dat'.format(directory))
test_labels = np.load('{0}/test_labels.dat'.format(directory))
print('Training labels shape: {0}'.format(train_labels.shape))
print('Testing labels shape:  {0}'.format(test_labels.shape))<jupyter_output>Training labels shape: (7326,)
Testing labels shape:  (2603,)<jupyter_text>Load Otsu's Binarization Threshold Data<jupyter_code>train_OBT_2d = np.load('{0}/train_OBT_2d.dat'.format(directory))
test_OBT_2d = np.load('{0}/test_OBT_2d.dat'.format(directory))
print('Otsu training shape: {0}'.format(train_OBT_2d.shape))
print('Otsu testing shape:  {0}'.format(test_OBT_2d.shape))<jupyter_output>Otsu training shape: (7326, 640)
Otsu testing shape:  (2603, 640)<jupyter_text>Load Adaptive Mean Threshold Data<jupyter_code>train_AMT_2d = np.load('{0}/train_AMT_2d.dat'.format(directory))
test_AMT_2d = np.load('{0}/test_AMT_2d.dat'.format(directory))
print('Adaptive mean training shape: {0}'.format(train_AMT_2d.shape))
print('Adaptive mean testing shape:  {0}'.format(test_AMT_2d.shape))<jupyter_output>Adaptive mean training shape: (7326, 640)
Adaptive mean testing shape:  (2603, 640)<jupyter_text>Load Adaptive Gaussian Threshold Data<jupyter_code>train_AGT_2d = np.load('{0}/train_AGT_2d.dat'.format(directory))
test_AGT_2d = np.load('{0}/test_AGT_2d.dat'.format(directory))
print('Adaptive Gaussian training shape: {0}'.format(train_AGT_2d.shape))
print('Adaptive Gaussian testing shape:  {0}'.format(test_AGT_2d.shape))<jupyter_output>Adaptive Gaussian training shape: (7326, 640)
Adaptive Gaussian testing shape:  (2603, 640)<jupyter_text>Load Principle Component Analysis Data<jupyter_code>train_PCA_2d = np.load('{0}/train_PCA_2d.dat'.format(directory))
test_PCA_2d = np.load('{0}/test_PCA_2d.dat'.format(directory))
print('PCA training shape: {0}'.format(train_PCA_2d.shape))
print('PCA testing shape:  {0}'.format(test_PCA_2d.shape))<jupyter_output>PCA training shape: (7326, 40)
PCA testing shape:  (2603, 40)<jupyter_text>Lists of Datasets<jupyter_code>names = ['OBT', 'AMT', 'AGT', 'PCA']
# extended names for readability
names_ext = ['Otsu', 'Adapt. Mean', 'Adapt. Gaussian', 'PCA']
l_train = [train_OBT_2d, train_AMT_2d, train_AGT_2d, train_PCA_2d]
l_test = [test_OBT_2d, test_AMT_2d, test_AGT_2d, test_PCA_2d]
overall_dict = {}<jupyter_output><empty_output><jupyter_text>Visualize Important Pixels<jupyter_code>img_dir = 'imgs'
if not os.path.exists(img_dir):
    os.makedirs(img_dir)
    
def important_pixels(mdl, save_img, ht=32, wd=20):
    '''
    source: http://scikit-learn.org/stable/auto_examples/ensemble/plot_forest_importances_faces. \
                            html#example-ensemble-plot-forest-importances-faces-py
    '''
    importances = mdl.feature_importances_
    try:
        importances = importances.reshape(ht,wd)

        # Plot pixel importances
        plt.matshow(importances, cmap=plt.cm.hot)
        plt.axis('off')
        plt.savefig('{0}/{1}'.format(img_dir, save_img), bbox_inches='tight')
    except:
        pass<jupyter_output><empty_output><jupyter_text>Model Folder<jupyter_code>mdl_dir = 'models'
if not os.path.exists(mdl_dir):
    os.makedirs(mdl_dir)<jupyter_output><empty_output><jupyter_text>Ada Boost Classifier<jupyter_code>from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier

# Attributes
# estimators_ : list of classifiers
# classes_ : array of shape = [n_classes]
# n_classes_ : int
# estimator_weights_ : array of floats
# estimator_errors_ : array of floats
# feature_importances_ : array of shape = [n_features]

ab_dict = {}

for i,dataset in enumerate(l_train):

    etr_clf = ExtraTreesClassifier(n_estimators=50, criterion='gini', max_depth=None, min_samples_split=4, 
                    min_samples_leaf=2, min_weight_fraction_leaf=0.0, max_features='sqrt', max_leaf_nodes=None, 
                    bootstrap=False, oob_score=False, n_jobs=-1, random_state=None, verbose=0, warm_start=False, 
                    class_weight=None)
    
    ab_clf = AdaBoostClassifier(base_estimator=etr_clf, n_estimators=50, learning_rate=0.1, 
                                 algorithm='SAMME.R', random_state=None)
    
    # cross-validation training score
    scores = cross_val_score(ab_clf, dataset, train_labels)
    
    # append results to dictionary
    ab_dict[('{0}'.format(names[i]))] = scores.mean()*100
    
    # models - save to disk for later use without re-running
    ab_mdl = ab_clf.fit(dataset, train_labels) 
    
    with open('{0}/ada_boost_{1}.pkl'.format(mdl_dir, names[i]), 'wb') as f:
        pickle.dump(ab_mdl, f)  
    
    # un-comment to get pixel importance images
    # important_pixels(ab_mdl, 'ada_boost_{0}'.format(names[i]))
overall_dict['ada_boost'] = ab_dict
ab_dict<jupyter_output><empty_output><jupyter_text>Bagging Classifier<jupyter_code>from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import ExtraTreesClassifier

# Attributes
# base_estimator_ : list of estimators
# estimators_ : list of estimators
# estimators_samples_ : list of arrays
# estimators_features_ : list of arrays
# classes_ : array of shape = [n_classes]
# n_classes_ : int or list
# oob_score_ : float
# oob_decision_function_ : array of shape = [n_samples, n_classes]

bag_dict = {}

for i,dataset in enumerate(l_train):
    
    etr_clf = ExtraTreesClassifier(n_estimators=50, criterion='gini', max_depth=None, min_samples_split=4, 
                    min_samples_leaf=2, min_weight_fraction_leaf=0.0, max_features='sqrt', max_leaf_nodes=None, 
                    bootstrap=False, oob_score=False, n_jobs=-1, random_state=None, verbose=0, warm_start=False, 
                    class_weight=None)
    
    bag_clf = BaggingClassifier(base_estimator=etr_clf, n_estimators=50, max_samples=1.0, max_features=1.0, 
                                bootstrap=True, bootstrap_features=False, oob_score=False, warm_start=False, 
                                n_jobs=-1, random_state=None, verbose=0)
    # cross-validation training score
    scores = cross_val_score(bag_clf, dataset, train_labels)
    
    # append results to dictionary
    bag_dict['{0}'.format(names[i])] = scores.mean()*100
    
    # models - save to disk for later use without re-running
    bag_mdl = bag_clf.fit(dataset, train_labels) 
    
    with open('{0}/bagging_{1}.pkl'.format(mdl_dir, names[i]), 'wb') as f:
        pickle.dump(bag_mdl, f)  
overall_dict['bagging'] = bag_dict
bag_dict<jupyter_output><empty_output><jupyter_text>Gradient Boosting Classifier - TAKES A LONG TIME TO RUN<jupyter_code>from sklearn.ensemble import GradientBoostingClassifier

# Attributes
# feature_importances_ : array, shape = [n_features]
# oob_improvement_ : array, shape = [n_estimators]
# train_score_ : array, shape = [n_estimators]
# loss_ : LossFunction
# init : BaseEstimator
# estimators_ : ndarray of DecisionTreeRegressor, shape = [n_estimators, loss_.K]
    
gb_dict = {}

n_estimators=300

# min_samples_split => 0.5%-1% of total values
min_samples_split = round(0.005*len(train_labels))

# min_samples_leaf => 50
min_samples_leaf = 50

# max_depth => between 5-8
max_depth = 6

# max_features => 'sqrt' as a rule of thumb to start
max_features = 'sqrt'

# subsample => 0.8 to start
subsample = 0.5

for i,dataset in enumerate(l_train):
    gb_clf = GradientBoostingClassifier(loss='deviance', learning_rate=0.1, n_estimators=n_estimators, 
                                    subsample=subsample, min_samples_split=min_samples_split, 
                                    min_samples_leaf=min_samples_leaf, max_depth=max_depth, init=None, 
                                    random_state=None, max_features=max_features, verbose=0, 
                                    max_leaf_nodes=None, warm_start=False, presort='auto')
    # cross-validation training score
    scores = cross_val_score(gb_clf, dataset, train_labels)
    
    # append results to dictionary
    gb_dict[('{0}'.format(names[i]))] = scores.mean()*100
    
    # models - save to disk for later use without re-running
    gb_mdl = gb_clf.fit(dataset, train_labels)
    
    with open('{0}/gradient_boost_{1}.pkl'.format(mdl_dir, names[i]), 'wb') as f:
        pickle.dump(gb_mdl, f)  
    
    # un-comment to get pixel importance images
    # important_pixels(gb_mdl, 'gradient_boost_{0}'.format(names[i]))
overall_dict['gradient_boost'] = gb_dict
gb_dict<jupyter_output><empty_output><jupyter_text>Extra Trees Classifier<jupyter_code>from sklearn.ensemble import ExtraTreesClassifier

etr_dict = {}

for i,dataset in enumerate(l_train):
    
    etr_clf = ExtraTreesClassifier(n_estimators=400, criterion='gini', max_depth=None, min_samples_split=4, 
                    min_samples_leaf=2, min_weight_fraction_leaf=0.0, max_features='sqrt', max_leaf_nodes=None, 
                    bootstrap=False, oob_score=False, n_jobs=-1, random_state=None, verbose=0, warm_start=False, 
                    class_weight=None)
    
    # cross-validation training score
    scores = cross_val_score(etr_clf, dataset, train_labels)
    
    # append results to dictionary
    etr_dict['{0}'.format(names[i])] = scores.mean()*100
    
    # models - save to disk for later use without re-running
    etr_mdl = etr_clf.fit(dataset, train_labels)
    
    with open('{0}/extra_trees_{1}.pkl'.format(mdl_dir, names[i]), 'wb') as f:
        pickle.dump(etr_mdl, f)  
    
    # un-comment to get pixel importance images
    # important_pixels(etr_mdl, 'extra_tree{0}'.format(names[i]))
overall_dict['extra_trees'] = etr_dict
etr_dict
etr_clf = ExtraTreesClassifier()
etr_mdl = etr_clf.fit(train_AMT_2d, train_labels)
print(classification_report(test_labels, etr_clf.predict(test_AMT_2d), digits=2))<jupyter_output>precision    recall  f1-score   support

          0       0.43      0.53      0.47       180
          1       0.65      0.83      0.73       509
          2       0.59      0.70      0.64       394
          3       0.46      0.48      0.47       295
          4       0.56      0.56      0.56       256
          5       0.55      0.52      0.53       231
          6       0.49      0.35      0.41       199
          7       0.65      0.53      0.59       202
          8       0.55      0.24      0.33       175
          9       0.51      0.27      0.35       162

avg / total       0.56      0.56      0.55      2603<jupyter_text>Random Forrest Classifier<jupyter_code>from sklearn.ensemble import RandomForestClassifier

# Attributes
# estimators_ : list of DecisionTreeClassifier
# classes_ : array of shape = [n_classes] or a list of such arrays
# n_classes_ : int or list
# n_features_ : int
# n_outputs_ : int
# feature_importances_ : array of shape = [n_features]
# oob_score_ : float
# oob_decision_function_ : array of shape = [n_samples, n_classes]
    
rf_dict = {}

for i,dataset in enumerate(l_train):
    rf_clf = RandomForestClassifier(n_estimators=150, criterion='gini', max_depth=None, min_samples_split=2, 
                                min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features='auto', 
                                max_leaf_nodes=None, bootstrap=True, oob_score=False, n_jobs=-1, random_state=None, 
                                verbose=0, warm_start=False, class_weight=None)

    # cross-validation training score
    scores = cross_val_score(rf_clf, dataset, train_labels)
    
    # append results to dictionary
    rf_dict['{0}'.format(names[i])] = scores.mean()*100
    
    # models - save to disk for later use without re-running
    rf_mdl = rf_clf.fit(dataset, train_labels)
    
    with open('{0}/random_forrest_{1}.pkl'.format(mdl_dir, names[i]), 'wb') as f:
        pickle.dump(rf_mdl, f)  
    
    # un-comment to get pixel importance images
    # important_pixels(rf_mdl, 'random_forrest{0}'.format(names[i]))
overall_dict['random_forrest'] = rf_dict
rf_dict<jupyter_output><empty_output><jupyter_text>KNN<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
    
knn_dict = {}

for i,dataset in enumerate(l_train):
    knn_clf = KNeighborsClassifier(n_neighbors=10, weights='uniform', algorithm='auto', leaf_size=30, p=2, 
                                   metric='minkowski', metric_params=None, n_jobs=-1)

    # cross-validation training score
    scores = cross_val_score(knn_clf, dataset, train_labels)
    
    # append results to dictionary
    knn_dict['{0}'.format(names[i])] = scores.mean()*100
    
    # models - save to disk for later use without re-running
    knn_mdl = knn_clf.fit(dataset, train_labels)
    
    with open('{0}/kneighbors_{1}.pkl'.format(mdl_dir, names[i]), 'wb') as f:
        pickle.dump(knn_mdl, f)  
overall_dict['kneighbors'] = knn_dict
knn_dict<jupyter_output><empty_output><jupyter_text>SVM Classifier<jupyter_code>from sklearn.svm import SVC

svc_dict = {}

C = [1, 10, 100]
gamma = [0.1, 0.01, 0.01, 0.001]

for i,dataset in enumerate(l_train):
    for c in C:
        for g in gamma:
            svc_clf = SVC(C=c, kernel='rbf', degree=3, gamma=g, coef0=0.0, shrinking=True, probability=False, 
                          tol=0.001, cache_size=1000, class_weight=None, verbose=False, max_iter=-1, 
                          decision_function_shape=None)

            # cross-validation training score
            scores = cross_val_score(svc_clf, dataset, train_labels)

            # append results to dictionary
            svc_dict[('{0}'.format(names[i]), c, g)] = scores.mean()*100

            # models - save to disk for later use without re-running
            svc_mdl = svc_clf.fit(dataset, train_labels)

            with open('{0}/svc_{1}-{2}-{3}.pkl'.format(mdl_dir, names[i], c, g), 'wb') as f:
                pickle.dump(svc_mdl, f)  
overall_dict['svc'] = svc_dict
svc_dict
list(overall_dict.items())
# get model names
mdls = [k for k,v in overall_dict.items()]

# get cross fold scores for each model
lst_cnts = [[v for k,v in overall_dict[mdl].items()] for mdl in mdls]
# colors
sns.set_palette("hot")
colors = sns.color_palette('hot',9)

# make figure
fig = plt.figure(figsize=(20,12))
ax = fig.add_subplot(111)

# spacing/size
width = 0.1
steps = [-0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4]
pos = []
for j in steps:
    temp = []
    for k in range(1,6):
        temp.append(k+j)
    pos.append(temp)
pos

# plots
for i in range(9):
    plt.bar(pos[i], lst_cnts[i], width=width, align='center', color=colors[i], alpha=0.8, edgecolor='w', 
            label=mdls[i])

# labels/titles
plt.legend(loc="best")
plt.xlabel('Threshold Technique')
plt.ylabel('CFV Score')
plt.title('Accuracy by Model and Threshold')

ticks = range(1,6)
plt.xlim(0.5,5.5)
plt.xticks(ticks, names_ext)
plt.ylim(0,100)

# remove border
ax.spines["top"].set_visible(False)  
ax.spines["right"].set_visible(False) 
ax.spines["bottom"].set_visible(False) 
ax.spines["left"].set_visible(False)

# show grid
ax.yaxis.grid(True) 

# plot that biddy
plt.tight_layout()
_ = plt.show()
df = pd.DataFrame(overall_dict)
df
# make figure
fig = plt.figure(figsize=(20,12))
ax = fig.add_subplot(111)

# plots
ax = sns.heatmap(df, annot=True, cmap="hot_r", alpha=0.8, linewidths=.1, vmin=0, vmax=100)

# labels/titles
plt.title('Accuracy by Model and Threshold')

# remove border
ax.spines["top"].set_visible(False)  
ax.spines["right"].set_visible(False) 
ax.spines["bottom"].set_visible(False) 
ax.spines["left"].set_visible(False)

# show grid
ax.yaxis.grid(False) 

# plot that biddy
plt.tight_layout()
_ = plt.show()
import re
'''
word = 'fubar'
reg_agt = re.compile(r'AGT')
if reg_agt.search(word) is not None:
  print 'matched'
'''
# l_test = [test_OBT_2d, test_AMT_2d, test_AGT_2d, test_PCA_2d]

with open('tuned_resuts.csv', 'a') as f:

    for fil in os.listdir('models'):
        if fil[-3:] == 'pkl':

            reg_agt = re.compile(r'AGT')
            if reg_agt.search(fil) is not None:
                #AGT
                with open('models/{0}'.format(fil), 'rb') as fid:
                    mdl = pickle.load(fid)
                    score = accuracy_score(test_labels, mdl.predict(test_AGT_2d))
                    f.write('{0}, {1}\n'.format(fil[:-4], score))
                    clf = None

            reg_agt = re.compile(r'AMT')
            if reg_agt.search(fil) is not None:
                #AMT
                with open('models/{0}'.format(fil), 'rb') as fid:
                    mdl = pickle.load(fid)
                    score = accuracy_score(test_labels, mdl.predict(test_AMT_2d))
                    f.write('{0}, {1}\n'.format(fil[:-4], score))
                    clf = None

            reg_pca = re.compile(r'PCA')
            if reg_pca.search(fil) is not None:
                #PCA
                with open('models/{0}'.format(fil), 'rb') as fid:
                    mdl = pickle.load(fid)
                    score = accuracy_score(test_labels, mdl.predict(test_PCA_2d))
                    f.write('{0}, {1}\n'.format(fil[:-4], score))
                    clf = None

            reg_obt = re.compile(r'OBT')
            if reg_obt.search(fil) is not None:
                #OBT
                with open('models/{0}'.format(fil), 'rb') as fid:
                    mdl = pickle.load(fid)
                    score = accuracy_score(test_labels, mdl.predict(test_OBT_2d))
                    f.write('{0}, {1}\n'.format(fil[:-4], score))
                    clf = None
df = pd.read_csv('tuned_resuts.csv',  names=['mdl', 'accuracy'])
df = df.sort_values('accuracy', ascending=False)
df.head(10)
fid = open('models/gradient_boost_AMT.pkl', 'rb')
mdl = pickle.load(fid)

print(classification_report(test_labels, mdl.predict(test_AMT_2d)))
from sklearn.ensemble import GradientBoostingClassifier

# Attributes
# feature_importances_ : array, shape = [n_features]
# oob_improvement_ : array, shape = [n_estimators]
# train_score_ : array, shape = [n_estimators]
# loss_ : LossFunction
# init : BaseEstimator
# estimators_ : ndarray of DecisionTreeRegressor, shape = [n_estimators, loss_.K]
    
gb_dict = {}

n_estimators=300

# min_samples_split => 0.5%-1% of total values
min_samples_split = round(0.005*len(train_labels))

# min_samples_leaf => 50
min_samples_leaf = 50

# max_depth => between 5-8
max_depth = 6

# max_features => 'sqrt' as a rule of thumb to start
max_features = 'sqrt'

# subsample => 0.8 to start
subsample = 0.5


gb_clf = GradientBoostingClassifier(loss='deviance', learning_rate=0.1, n_estimators=n_estimators, 
                                subsample=subsample, min_samples_split=min_samples_split, 
                                min_samples_leaf=min_samples_leaf, max_depth=max_depth, init=None, 
                                random_state=None, max_features=max_features, verbose=0, 
                                max_leaf_nodes=None, warm_start=False, presort='auto')

# cross-validation training score
scores = cross_val_score(gb_clf, train_AMT_2d, train_labels)

gb_mdl = gb_clf.fit(train_AMT_2d, train_labels)
p_pred = gb_mdl.predict_proba(test_AMT_2d)
y_test_predicted = gb_mdl.predict(test_AMT_2d)
roc_auc_score(test_labels, p_pred[:, 1])
false_positive_rate, true_positive_rate, thresholds = roc_curve(test_labels, p_pred[:, 1])

print(roc_auc_score(test_labels, mdl.predict(test_AMT_2d)))
fid.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12299300123507616, 'fraction_numerical': 0.027634829147797448, 'mean_word_length': 3.3299910873440286, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '54738', 'license': 'MIT', 'n_tokens_mistral': 7368, 'n_tokens_neox': 6910, 'n_words': 1309, 'path': 'tuned_models.ipynb', 'repo_name': 'bryantbiggs/mcnulty-03', 'size': '19432'}