<jupyter_start><jupyter_text>Play with keras model<jupyter_code>%pylab inline
import numpy as np
import csv
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.utils import np_utils
from sklearn import metrics
from sklearn.cross_validation import KFold
from sklearn.preprocessing import StandardScaler
from sklearn import utils
from preprocess import loadPreData
from preprocess import scoreAUC
subm_folder = './submissions'
'''
    This demonstrates how to reach a 0.80 ROC AUC score (local 4-fold validation)
    in the Kaggle Nile virus prediction challenge. 

    The model trains in a few seconds on CPU.
'''
data = loadPreData(features=True)
X = data['train'].get_values().astype(float)
y = data['labels']
Y = np_utils.to_categorical(y)
input_dim = X.shape[1]
output_dim = 2
def build_model(input_dim, output_dim):
    model = Sequential()
    model.add(Dense(32, input_dim=input_dim, kernel_initializer='random_uniform'))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))

    model.add(Dense(32, kernel_initializer='random_uniform'))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    
    model.add(Dense(32, kernel_initializer='random_uniform'))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))

    model.add(Dense(output_dim, kernel_initializer='random_uniform'))

    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
    return model
def testModel():    
    def logit(x):
        return 1/(1+np.exp(-x))

    X = (np.random.rand(5000,5))
    covs = np.tile(100.*(1.-2.*np.random.rand(5)),(5000,1))
    y = np.sum(X*covs,axis=1)

    y =  (logit(y)>0.5).astype(float)
    print y.shape
    y = np_utils.to_categorical(y)
    
    X_valid = np.random.rand(1000,5)
    
    y_valid = np.sum(covs[:1000,:]*X_valid,axis=1)
    y_valid =  (logit(y_valid)>0.5).astype(float)
    y_valid = np_utils.to_categorical(y_valid)
    
    model = build_model(5, 2)
    print 'Data input shape {}, data output shape {}'.format(X.shape,y.shape)
    model.fit(X, y, nb_epoch=10, batch_size=32, validation_data=(X_valid, y_valid), verbose=True)
    return {'model':model,'X_valid':X_valid,'y_valid':y_valid,'X_train':X,'y_train':y}
res = testModel()
model = res['model']
X_train,X_test = res['X_train'],res['X_valid']
y_train,y_test = res['y_train'],res['y_valid']
y_train,y_test = y_train[:,1].astype(bool),y_test[:,1].astype(bool)
probs = model.predict_proba(X_train)
auc = scoreAUC(y_train,probs)
plt.plot(auc['fpr'],auc['tpr'],label='Train');

probs = model.predict_proba(X_test)
auc = scoreAUC(y_test,probs)
plt.plot(auc['fpr'],auc['tpr'],label='Validation');

plt.legend();

plt.plot([0,1.],[0,1.1],'k--');
plt.xlabel('False positive rate');
plt.ylabel('True positive rate');


plt.title('AUC: {:.3f}'.format(auc['score']));
#class_weight = {0 : 1.,1: 50.}

nb_folds = 4
kfolds = KFold(len(y), nb_folds)
av_roc = 0.
f = 0
for train, valid in kfolds:
    print('---'*20)
    print('Fold', f)
    print('---'*20)
    f += 1
    X_train = X[train]
    X_valid = X[valid]
    Y_train = Y[train]
    Y_valid = Y[valid]
    y_valid = y[valid]
    print("fold has {} WNV present".format(np.sum(Y_train[:,1])))
    class_weight = utils.class_weight.compute_class_weight('balanced', np.unique(y[train]), y[train])
    print("Class weight: {}".format(class_weight))
    print("Building model...")
    model = build_model(input_dim, output_dim)

    print("Training model...")

    model.fit(X_train, Y_train, epochs=20, batch_size=32, validation_data=(X_valid, Y_valid), 
              class_weight=class_weight,verbose=True)
    valid_preds = model.predict_proba(X_valid)
    valid_preds = valid_preds[:, 1]
    roc = metrics.roc_auc_score(y_valid, valid_preds)
    print("ROC:", roc)
    av_roc += roc

print('Average ROC:', av_roc/nb_folds)
np.sum(model.predict_proba(X_valid)[:,1])
class_weight = utils.class_weight.compute_class_weight('balanced', np.unique(y), y)
Y = np_utils.to_categorical(y)
model = build_model(input_dim, output_dim)
model.fit(X, Y, nb_epoch=20, batch_size=32, verbose=1,class_weight=class_weight)
probs = model.predict_proba(X)
auc = scoreAUC(y,probs)
plt.plot(auc['fpr'],auc['tpr'],label='Train');




plt.legend();

plt.plot([0,1.],[0,1.],'k--');
plt.xlabel('False positive rate');
plt.ylabel('True positive rate');


plt.title('AUC: {:.3f}'.format(auc['score']));
X_sub = data['test'].get_values()
subs = model.predict_proba(X_sub)

np.sum(data['labels']==1)/float(data['labels'].size)
plt.hist(subs[:,1]);
#prepare submission
import pandas as pd
import os.path
ss = pd.DataFrame(columns=['Id','WnvPresent'])
ss['WnvPresent'] = pd.Series(subs[:,1])
ss['Id'] = data['test'].index + 1
ss.to_csv(os.path.join(subm_folder, 'keras_submission_1.csv'), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17142274765593152, 'fraction_numerical': 0.020383204239706482, 'mean_word_length': 4.912048192771084, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '253374', 'license': 'MIT', 'n_tokens_mistral': 1937, 'n_tokens_neox': 1848, 'n_words': 322, 'path': 'keras_play.ipynb', 'repo_name': 'sempwn/west-nile-virus', 'size': '4906'}