<jupyter_start><jupyter_text>cosine similarity<jupyter_code>from __future__      import division
from IPython.display import display
from matplotlib      import pyplot as plt
%matplotlib inline

import numpy  as np
import pandas as pd
import random, sys, os, re

from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cross_validation import train_test_split

from sklearn.cross_validation import StratifiedKFold
from sklearn.grid_search      import RandomizedSearchCV, GridSearchCV
from sklearn.cross_validation import cross_val_predict, permutation_test_score
SEED   = 97
scale  = False 
minmax = False
norm   = False
nointercept = True
engineering = False

N_CLASSES = 2

submission_filename = "../submissions/submission_cosine_similarity.csv"<jupyter_output><empty_output><jupyter_text>Load the training data<jupyter_code>from load_blood_data import load_blood_data

y_train, X_train = load_blood_data(train=True, SEED   = SEED, 
                                               scale  = scale,
                                               minmax = minmax,
                                               norm   = norm,
                                               nointercept = nointercept,
                                               engineering = engineering)<jupyter_output><empty_output><jupyter_text>Train/Valid split<jupyter_code>X_train, X_valid, y_train, y_valid = train_test_split(
    X_train, y_train, test_size=0.33, random_state=SEED)<jupyter_output><empty_output><jupyter_text>Fit the "model"<jupyter_code>from sklearn.metrics import confusion_matrix
y_true = y_valid
scores = []
for how_many in np.arange(50)+1:
    y_pred = []
    for i in range(len(X_valid)):
        top_how_many = cosine_similarity(X_valid.values[i:i+1], X_train.values).argsort()[0][::-1][:how_many]
        if np.mean([y_train[ii] for ii in top_how_many]) < 0.51:
            y_pred.append(0)
        else:
            y_pred.append(1)
    cm = confusion_matrix(y_true, y_pred, labels=None)
    scores.append(round(np.trace(cm)/float(np.sum(cm)),4))
    
best_k = np.argmax(scores)+1
print("best_k {}".format(best_k))<jupyter_output>best_k 19<jupyter_text>Training set predictions<jupyter_code>how_many = best_k
y_pred = []
for i in range(len(X_valid)):
    top_how_many = cosine_similarity(X_valid.values[i:i+1], X_train.values).argsort()[0][::-1][:how_many]
    if np.mean([y_train[ii] for ii in top_how_many]) < 0.5:
        y_pred.append(0)
    else:
        y_pred.append(1)
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_true, y_pred, labels=None)
print cm

try:
    from sklearn_utilities import plot_confusion_matrix
except:
    import imp, os
    util = imp.load_source('sklearn_utilities', os.path.expanduser('~/Dropbox/Python/sklearn_utilities.py'))
    from sklearn_utilities import plot_confusion_matrix


plot_confusion_matrix(cm, ['Did not Donate','Donated'])

accuracy = round(np.trace(cm)/float(np.sum(cm)),4)
misclass = 1 - accuracy
print("Accuracy {}, mis-class rate {}".format(accuracy,misclass))
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.metrics import log_loss
from sklearn.metrics import f1_score

fpr, tpr, thresholds = roc_curve(y_true, y_pred, pos_label=None)


plt.figure(figsize=(10,6))
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr, tpr)

plt.text(x=0.6,y=0.4,s="AUC         {:.4f}"\
         .format(roc_auc_score(y_true, y_pred, average='macro')),
        fontsize=16)

plt.text(x=0.6,y=0.3,s="accuracy {:.2f}%"\
         .format(accuracy*100),
        fontsize=16)

plt.text(x=0.6,y=0.2,s="LogLoss   {:.4f}"\
         .format(log_loss(y_true, y_pred)),
        fontsize=16)

plt.text(x=0.6,y=0.1,s="f1            {:.4f}"\
         .format(f1_score(y_true, y_pred)),
        fontsize=16)

plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.show()<jupyter_output><empty_output><jupyter_text>--------------------------------------------------------------------------------------------  Test Set Predictions  Re-read the training data<jupyter_code>X_train = None; X_valid = None
y_train = None; y_valid = None
from load_blood_data import load_blood_data

y_train, X_train = load_blood_data(train=True, SEED   = SEED, 
                                               scale  = scale,
                                               minmax = minmax,
                                               norm   = norm,
                                               nointercept = nointercept,
                                               engineering = engineering)<jupyter_output><empty_output><jupyter_text>Read the test data<jupyter_code>from load_blood_data import load_blood_data

X_test, IDs = load_blood_data(train=False, SEED   = SEED, 
                                           scale  = scale,
                                           minmax = minmax,
                                           norm   = norm,
                                           nointercept = nointercept,
                                           engineering = engineering)<jupyter_output><empty_output><jupyter_text>Predict the test set with the "model"<jupyter_code>how_many = best_k
y_pred = []
for i in range(len(X_test)):
    top_how_many = cosine_similarity(X_test.values[i:i+1], X_train.values).argsort()[0][::-1][:how_many]
    if np.mean([y_train[ii] for ii in top_how_many]) < 0.5:
        y_pred.append(0)
    else:
        y_pred.append(1)
donate_probs = [0.65 if x>0 else 1-0.65 for x in y_pred]
print(y_pred[:10])
print(donate_probs[:10])<jupyter_output>[0, 0, 0, 0, 1, 1, 0, 0, 0, 0]
[0.35, 0.35, 0.35, 0.35, 0.65, 0.65, 0.35, 0.35, 0.35, 0.35]<jupyter_text>Create the submission file<jupyter_code>assert len(IDs)==len(donate_probs)

f = open(submission_filename, "w")

f.write(",Made Donation in March 2007\n")
for ID, prob in zip(IDs, donate_probs):
    f.write("{},{}\n".format(ID,prob))
    
f.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12926421404682276, 'fraction_numerical': 0.022408026755852843, 'mean_word_length': 2.624848484848485, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '425561', 'license': 'MIT', 'n_tokens_mistral': 2112, 'n_tokens_neox': 1985, 'n_words': 387, 'path': 'cosine similarity.ipynb', 'repo_name': 'grfiv/predict-blood-donations', 'size': '5980'}