<jupyter_start><jupyter_text>Day 57, 100  Import modules<jupyter_code>%matplotlib inline
from __future__ import division
import sys
import os
os.environ['MKL_THREADING_LAYER']='GNU'
sys.path.append('../')
from Modules.Basics import *
from Modules.Class_Basics import *<jupyter_output><empty_output><jupyter_text>Options<jupyter_code>classTrainFeatures = ['DER_mass_MMC', 'DER_mass_transverse_met_lep', 'DER_mass_vis', 'DER_pt_h', 'DER_deltaeta_jet_jet', 'DER_mass_jet_jet', 'DER_prodeta_jet_jet', 'DER_deltar_tau_lep', 'DER_pt_tot', 'DER_sum_pt', 'DER_pt_ratio_lep_tau', 'DER_met_phi_centrality', 'DER_lep_eta_centrality', 'PRI_met_pt', 'PRI_met_sumet', 'PRI_jet_num', 'PRI_jet_all_pt', 'PRI_tau_px', 'PRI_tau_py', 'PRI_tau_pz', 'PRI_lep_px', 'PRI_lep_pz', 'PRI_jet_leading_px', 'PRI_jet_leading_py', 'PRI_jet_leading_pz', 'PRI_jet_subleading_px', 'PRI_jet_subleading_py', 'PRI_jet_subleading_pz', 'PRI_met_px', 'PRI_met_py']
inputPipe, outputPipe = getPreProcPipes(normIn=True)
classModel = 'modelSwish-7-100'
varSet = "basic_rot_features"

nSplits = 10
ensembleSize = 10
ensembleMode = 'loss'

maxEpochs = 200
compileArgs = {'loss':'binary_crossentropy', 'optimizer':'adam', 'depth':7, 'width':50}
trainParams = {'epochs' : 1, 'batch_size' : 256, 'verbose' : 0}
modelParams = {'version':classModel, 'nIn':len(classTrainFeatures), 'compileArgs':compileArgs}

print "\nTraining on", len(classTrainFeatures), "features:", [var for var in classTrainFeatures]<jupyter_output>Training on 30 features: ['DER_mass_MMC', 'DER_mass_transverse_met_lep', 'DER_mass_vis', 'DER_pt_h', 'DER_deltaeta_jet_jet', 'DER_mass_jet_jet', 'DER_prodeta_jet_jet', 'DER_deltar_tau_lep', 'DER_pt_tot', 'DER_sum_pt', 'DER_pt_ratio_lep_tau', 'DER_met_phi_centrality', 'DER_lep_eta_centrality', 'PRI_met_pt', 'PRI_met_sumet', 'PRI_jet_num', 'PRI_jet_all_pt', 'PRI_tau_px', 'PRI_tau_py', 'PRI_tau_pz', 'PRI_lep_px', 'PRI_lep_pz', 'PRI_jet_leading_px', 'PRI_jet_leading_py', 'PRI_jet_leading_pz', 'PRI_jet_subleading_px', 'PRI_jet_subleading_py', 'PRI_jet_subleading_pz', 'PRI_met_px', 'PRI_met_py']<jupyter_text>Import data<jupyter_code>trainData = h5py.File(dirLoc + 'train.hdf5', "r+")
valData = h5py.File(dirLoc + 'val.hdf5', "r+")<jupyter_output><empty_output><jupyter_text>Determine LR<jupyter_code>lrFinder = batchLRFindClassifier(trainData, nSplits, getClassifier, modelParams, trainParams, lrBounds=[1e-5,1e-1], trainOnWeights=True, verbose=0)
compileArgs['lr'] = 5e-3<jupyter_output><empty_output><jupyter_text>Train classifier<jupyter_code>results, histories = batchTrainClassifier(trainData, nSplits, getClassifier, modelParams, trainParams, patience=100, cosAnnealMult=2, trainOnWeights=True, maxEpochs=maxEpochs, verbose=1)<jupyter_output>Using cosine annealing
Training using weights
Running fold 1 / 10
2 classes found, running in binary mode

1 New best found: 3.90026026286e-05
2 New best found: 3.64956325804e-05
3 New best found: 3.54543244003e-05
5 New best found: 3.49591821861e-05
6 New best found: 3.43826850992e-05
7 New best found: 3.41430344579e-05
12 New best found: 3.38258467696e-05
13 New best found: 3.34889353204e-05
14 New best found: 3.34066312179e-05
15 New best found: 3.34044028362e-05
25 New best found: 3.32746661543e-05
27 New best found: 3.29879968606e-05
28 New best found: 3.29554846548e-05
29 New best found: 3.29362520796e-05
30 New best found: 3.29258239473e-05
31 New best found: 3.29211229614e-05
52 New best found: 3.289924318e-05
54 New best found: 3.28194709491e-05
55 New best found: 3.27622555301e-05
57 New best found: 3.27451009578e-05
90 New best found: 3.2726387679e-05
Early stopping after 190 epochs
Score is: {'loss': 3.2726387679033933e-05, 'AUC': 0.10322470250595406, 'wAUC': 0.066766967549[...]<jupyter_text>Construct ensemble<jupyter_code>with open('train_weights/resultsFile.pkl', 'r') as fin:   
    results = pickle.load(fin)
ensemble, weights = assembleEnsemble(results, ensembleSize, ensembleMode, compileArgs)<jupyter_output>Choosing ensemble by loss
Model 0 is 2 with loss = 3.119907727e-05
Model 1 is 9 with loss = 3.13069018686e-05
Model 2 is 8 with loss = 3.16576976812e-05
Model 3 is 3 with loss = 3.18712371057e-05
Model 4 is 1 with loss = 3.20804762087e-05
Model 5 is 5 with loss = 3.21852991782e-05
Model 6 is 4 with loss = 3.22806073461e-05
Model 7 is 7 with loss = 3.26868152686e-05
Model 8 is 0 with loss = 3.2726387679e-05
Model 9 is 6 with loss = 3.32842895887e-05<jupyter_text>Response on development data<jupyter_code>batchEnsemblePredict(ensemble, weights, trainData, ensembleSize=10, verbose=1)
print 'Training ROC AUC: unweighted {}, weighted {}'.format(roc_auc_score(getFeature('targets', trainData), getFeature('pred', trainData)),
                                                            roc_auc_score(getFeature('targets', trainData), getFeature('pred', trainData), sample_weight=getFeature('weights', trainData)))<jupyter_output>Training ROC AUC: unweighted 0.903684303041, weighted 0.939810797271<jupyter_text>Response on val data<jupyter_code>batchEnsemblePredict(ensemble, weights, valData, ensembleSize=10, verbose=1)
print 'Testing ROC AUC: unweighted {}, weighted {}'.format(roc_auc_score(getFeature('targets', valData), getFeature('pred', valData)),
                                                           roc_auc_score(getFeature('targets', valData), getFeature('pred', valData), sample_weight=getFeature('weights', valData)))<jupyter_output>Testing ROC AUC: unweighted 0.901246233133, weighted 0.934826844758<jupyter_text>Evaluation  Import in dataframe<jupyter_code>valframe = convertToDF(valData)
sigVal = (valframe.gen_target == 1)
bkgVal = (valframe.gen_target == 0)<jupyter_output><empty_output><jupyter_text>MVA distributions<jupyter_code>getClassPredPlot([valframe[bkgVal], valframe[sigVal]], weightName='gen_weight')
valframe['ams'] = amsScan(valframe)[0]
bests = foldAMSScan(valframe, 10)
def scoreTest(ensemble, weights):
    testData = h5py.File(dirLoc + 'testing.hdf5', "r+")
    batchEnsemblePredict(ensemble, weights, testData, ensembleSize=10, verbose=1)

def saveTest(cut, name):
    testData = h5py.File(dirLoc + 'testing.hdf5', "r+")
    
    data = pandas.DataFrame()
    data['EventId'] = getFeature('EventId', testData)
    data['pred_class'] = getFeature('pred', testData)
    
    data['Class'] = 'b'
    data.loc[data.pred_class >= cut, 'Class'] = 's'

    data.sort_values(by=['pred_class'], inplace=True)
    data['RankOrder']=range(1, len(data)+1)
    data.sort_values(by=['EventId'], inplace=True)

    print dirLoc + name + '_test.csv'
    data.to_csv(dirLoc + name + '_test.csv', columns=['EventId', 'RankOrder', 'Class'], index=False)
scoreTest(ensemble, weights)
def saveTest(cut, name):
    testData = h5py.File(dirLoc + 'testing.hdf5', "r+")
    
    data = pandas.DataFrame()
    data['EventId'] = getFeature('EventId', testData)
    data['pred_class'] = getFeature('pred', testData)
    
    data['Class'] = 'b'
    data.loc[data.pred_class >= cut, 'Class'] = 's'

    data.sort_values(by=['pred_class'], inplace=True)
    data['RankOrder']=range(1, len(data)+1)
    data.sort_values(by=['EventId'], inplace=True)

    print dirLoc + name + '_test.csv'
    data.to_csv(dirLoc + name + '_test.csv', columns=['EventId', 'RankOrder', 'Class'], index=False)
saveTest(0.957172385359, 'Day_5')
!kaggle competitions submit -c higgs-boson -f ../Data/Day_5_test.csv -m"7-50"<jupyter_output>Successfully submitted to Higgs Boson Machine Learning Challenge
{'fraction_non_alphanumeric': 0.1340662287169862, 'fraction_numerical': 0.09076283684139966, 'mean_word_length': 6.34251968503937, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '494919', 'license': 'MIT', 'n_tokens_mistral': 3252, 'n_tokens_neox': 2823, 'n_words': 561, 'path': 'Classifiers/Day5/Day_5_7-50.ipynb', 'repo_name': 'GilesStrong/Kaggle_HiggsML', 'size': '7459'}