<jupyter_start><jupyter_text>Day 2batch size 256 lr 1e-3, weighted  Import modules<jupyter_code>%matplotlib inline
from __future__ import division
import sys
import os
os.environ['MKL_THREADING_LAYER']='GNU'
sys.path.append('../')
from Modules.Basics import *
from Modules.Class_Basics import *<jupyter_output>/home/giles/anaconda2/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
/home/giles/anaconda2/lib/python2.7/site-packages/statsmodels/compat/pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.
  from pandas.core import datetools
Using TensorFlow backend.<jupyter_text>Options<jupyter_code>classTrainFeatures = basic_features
classModel = 'modelSwish'
varSet = "basic_features"

nSplits = 10
ensembleSize = 10
ensembleMode = 'loss'

maxEpochs = 200
compileArgs = {'loss':'binary_crossentropy', 'optimizer':'adam'}
trainParams = {'epochs' : 1, 'batch_size' : 256, 'verbose' : 0}
modelParams = {'version':classModel, 'nIn':len(classTrainFeatures), 'compileArgs':compileArgs}

print "\nTraining on", len(classTrainFeatures), "features:", [var for var in classTrainFeatures]<jupyter_output>Training on 30 features: ['DER_mass_MMC', 'DER_mass_transverse_met_lep', 'DER_mass_vis', 'PRI_tau_pt', 'DER_deltar_tau_lep', 'PRI_jet_leading_eta', 'PRI_lep_eta', 'PRI_tau_eta', 'DER_lep_eta_centrality', 'DER_mass_jet_jet', 'DER_deltaeta_jet_jet', 'DER_pt_tot', 'DER_met_phi_centrality', 'PRI_jet_num', 'PRI_met', 'DER_pt_h', 'DER_pt_ratio_lep_tau', 'DER_sum_pt', 'PRI_jet_leading_pt', 'PRI_jet_subleading_eta', 'PRI_jet_all_pt', 'PRI_met_sumet', 'DER_prodeta_jet_jet', 'PRI_lep_pt', 'PRI_jet_subleading_pt', 'PRI_met_phi', 'PRI_jet_leading_phi', 'PRI_lep_phi', 'PRI_jet_subleading_phi', 'PRI_tau_phi']<jupyter_text>Import data<jupyter_code>trainData = h5py.File(dirLoc + 'train.hdf5', "r+")
valData = h5py.File(dirLoc + 'val.hdf5', "r+")<jupyter_output><empty_output><jupyter_text>Determine LR<jupyter_code>lrFinder = batchLRFindClassifier(trainData, nSplits, getClassifier, modelParams, trainParams, lrBounds=[1e-5,1e-2], trainOnWeights=True, verbose=0)
compileArgs['lr'] = 1e-3<jupyter_output><empty_output><jupyter_text>Train classifier<jupyter_code>results, histories = batchTrainClassifier(trainData, nSplits, getClassifier, modelParams, trainParams, patience=200, cosAnnealMult=2, trainOnWeights=True, maxEpochs=maxEpochs, verbose=1)<jupyter_output>Using cosine annealing
Training using weights
Running fold 1 / 10
2 classes found, running in binary mode

1 New best found: 0.0224863965639
2 New best found: 0.0179314360422
3 New best found: 0.0176988300299
4 New best found: 0.0170002694676
5 New best found: 0.0167144588916
6 New best found: 0.0165935251367
7 New best found: 0.0165781418649
8 New best found: 0.0163654108564
9 New best found: 0.0161831617113
10 New best found: 0.0160696771942
11 New best found: 0.0159486996469
12 New best found: 0.015885364874
13 New best found: 0.0158741805679
14 New best found: 0.0158144522649
15 New best found: 0.0158104772137
17 New best found: 0.0156520069599
18 New best found: 0.015604684526
20 New best found: 0.0155388842078
23 New best found: 0.015512171408
24 New best found: 0.0154280940226
26 New best found: 0.0153812382108
27 New best found: 0.0153608034782
29 New best found: 0.0153538191776
40 New best found: 0.0153200944867
41 New best found: 0.0152659927333
46 New best found: 0.015226774[...]<jupyter_text>Construct ensemble<jupyter_code>with open('train_weights/resultsFile.pkl', 'r') as fin:   
    results = pickle.load(fin)
ensemble, weights = assembleEnsemble(results, ensembleSize, ensembleMode, compileArgs)<jupyter_output>Choosing ensemble by loss
Model 0 is 5 with loss = 0.014789282591220402
Model 1 is 0 with loss = 0.014955854490226815
Model 2 is 2 with loss = 0.014979931187169697
Model 3 is 7 with loss = 0.015017185204217279
Model 4 is 9 with loss = 0.015140930447543078
Model 5 is 3 with loss = 0.015180114317463776
Model 6 is 1 with loss = 0.015184319923144913
Model 7 is 8 with loss = 0.015229138184619335
Model 8 is 4 with loss = 0.015272155381626655
Model 9 is 6 with loss = 0.015414068611701925<jupyter_text>Response on development data<jupyter_code>batchEnsemblePredict(ensemble, weights, trainData, ensembleSize=10, verbose=1)
print 'Training ROC AUC: unweighted {}, weighted {}'.format(roc_auc_score(getFeature('targets', trainData), getFeature('pred', trainData)),
                                                            roc_auc_score(getFeature('targets', trainData), getFeature('pred', trainData), sample_weight=getFeature('weights', trainData)))<jupyter_output>Training ROC AUC: unweighted 0.897035423378, weighted 0.935845245322<jupyter_text>Response on val data<jupyter_code>batchEnsemblePredict(ensemble, weights, valData, ensembleSize=10, verbose=1)
print 'Testing ROC AUC: unweighted {}, weighted {}'.format(roc_auc_score(getFeature('targets', valData), getFeature('pred', valData)),
                                                           roc_auc_score(getFeature('targets', valData), getFeature('pred', valData), sample_weight=getFeature('weights', valData)))<jupyter_output>Testing ROC AUC: unweighted 0.895394187597, weighted 0.932187238411<jupyter_text>Evaluation  Import in dataframe<jupyter_code>def convertToDF(datafile, columns={'gen_target', 'gen_weight', 'pred_class'}, nLoad=-1):
    data = pandas.DataFrame()
    data['gen_target'] = getFeature('targets', datafile, nLoad)
    data['gen_weight'] = getFeature('weights', datafile, nLoad)
    data['pred_class'] = getFeature('pred', datafile, nLoad)
    print len(data), "candidates loaded"
    return data
devData = convertToDF(trainData)
valData = convertToDF(valData)
sigVal = (valData.gen_target == 1)
bkgVal = (valData.gen_target == 0)<jupyter_output><empty_output><jupyter_text>MVA distributions<jupyter_code>getClassPredPlot([valData[bkgVal], valData[sigVal]], weightName='gen_weight')
amsScan(valData)
scoreTest(ensemble, weights, classTrainFeatures, 0.0225139781832695, 'Model_0_Basic_Features_256_1e-3_swish_mult2_200E_weighted_comp')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10839981229469732, 'fraction_numerical': 0.11747223525731268, 'mean_word_length': 6.630071599045346, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '486414', 'license': 'MIT', 'n_tokens_mistral': 2745, 'n_tokens_neox': 2254, 'n_words': 522, 'path': 'Classifiers/Day2/Day_2_Basic_Features_256_1e-3_swish_mult2_200E_weighted.ipynb', 'repo_name': 'GilesStrong/Kaggle_HiggsML', 'size': '6393'}