<jupyter_start><jupyter_text>Import modules<jupyter_code>%matplotlib inline
from __future__ import division
import sys
import os
os.environ['MKL_THREADING_LAYER']='GNU'
sys.path.append('../../')
from Modules.Basics import *
from Modules.Class_Basics import *

from scipy.stats import ks_2samp
dirLoc = '../../Data/'
def getCuts(nJets):
    valData = h5py.File(dirLoc + 'val_' + str(nJets) + '.hdf5', "r+")
    valframe = convertToDF(valData)
    bests = foldAMSScan(valframe, 3)
getCuts(1)
cuts = [0.974683394325156,
        0.9280973880451805,
        0.9617854822776671]
def saveTest(cuts, name):
    data = pandas.DataFrame()
    
    for i, cut in enumerate(cuts):
        tmp = pandas.DataFrame()
        testData = h5py.File(dirLoc + 'testing_' + str(i) + '.hdf5', "r")
        
        tmp['EventId'] = getFeature('EventId', testData)
        tmp['pred_class'] = getFeature('pred', testData)
    
        tmp['Class'] = 'b'
        tmp.loc[tmp.pred_class >= cut, 'Class'] = 's'
        
        data = data.append(tmp, ignore_index = True)

    data.sort_values(by=['pred_class'], inplace=True)
    data['RankOrder']=range(1, len(data)+1)
    data.sort_values(by=['EventId'], inplace=True)

    print dirLoc + name + '_test.csv'
    print len(data), len(list(set(data['EventId'])))
    data.to_csv(dirLoc + name + '_test.csv', columns=['EventId', 'RankOrder', 'Class'], index=False)
def plotSplitDist(cut, nJets):
    test = pandas.DataFrame()
    testData = h5py.File(dirLoc + 'testing_' + str(nJets) + '.hdf5', "r")

    test['pred_class'] = getFeature('pred', testData)

    test['Class'] = 'b'
    test.loc[test.pred_class >= cut, 'Class'] = 's'
    
    val = pandas.DataFrame()
    valData = h5py.File(dirLoc + 'val_' + str(nJets) + '.hdf5', "r")

    val['pred_class'] = getFeature('pred', valData)

    val['Class'] = 'b'
    val.loc[val.pred_class >= cut, 'Class'] = 's'
    
    print "signal agreement:", ks_2samp(test[test.Class == 's']['pred_class'], val[val.Class == 's']['pred_class'])[1]
    print "background agreement:", ks_2samp(test[test.Class == 'b']['pred_class'], val[val.Class == 'b']['pred_class'])[1]
    
    getClassPredPlot([test[test.Class == 's'], test[test.Class == 'b'],
                      val[val.Class == 's'], val[val.Class == 'b']], 
                     labels=['Testing Bkg', 'Testing Sig', 'Val Bkg', 'Val Sig'])
def plotDist(cuts):
    test = getData(cuts, 'testing')
    val = getData(cuts, 'val')
    
    print "signal agreement:", ks_2samp(test[test.Class == 's']['pred_class'], val[val.Class == 's']['pred_class'])[1]
    print "background agreement:", ks_2samp(test[test.Class == 'b']['pred_class'], val[val.Class == 'b']['pred_class'])[1]
    
    getClassPredPlot([test[test.Class == 's'], test[test.Class == 'b'],
                      val[val.Class == 's'], val[val.Class == 'b']], 
                     labels=['Testing Bkg', 'Testing Sig', 'Val Bkg', 'Val Sig'])
def getData(cuts, mode):
    data = pandas.DataFrame()
    
    for i, cut in enumerate(cuts):
        tmp = pandas.DataFrame()
        datafile = h5py.File(dirLoc + mode + '_' + str(i) + '.hdf5', "r+")
        
        tmp['pred_class'] = getFeature('pred', datafile)
        
        if mode == 'val':
            tmp['gen_target'] = getFeature('targets', datafile)
            tmp['gen_weight'] = getFeature('weights', datafile)
    
        tmp['Class'] = 'b'
        tmp.loc[tmp.pred_class >= cut, 'Class'] = 's'
        
        data = data.append(tmp, ignore_index = True)

    return data<jupyter_output><empty_output><jupyter_text>Validation data<jupyter_code>valData = getData(cuts, 'val')
AMS(np.sum(valData[(valData.gen_target == 1) & (valData.Class == 's')]['gen_weight']), np.sum(valData[(valData.gen_target == 0) & (valData.Class == 's')]['gen_weight']))
getClassPredPlot([valData[(valData.gen_target == 0)], valData[(valData.gen_target == 1)]], weightName='gen_weight')<jupyter_output><empty_output><jupyter_text>Testing data<jupyter_code>for i, cut in enumerate(cuts):
    plotSplitDist(cut, i)
plotDist(cuts)
saveTest(cuts, 'SplitJets')
!kaggle competitions submit -c higgs-boson -f ../../Data/SplitJets_test.csv -m"Day3 redux"<jupyter_output>Successfully submitted to Higgs Boson Machine Learning Challenge
{'fraction_non_alphanumeric': 0.18583862620559868, 'fraction_numerical': 0.01834862385321101, 'mean_word_length': 3.5524625267665955, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '424212', 'license': 'MIT', 'n_tokens_mistral': 1581, 'n_tokens_neox': 1509, 'n_words': 286, 'path': 'Classifiers/Day3/Day_3_JetSplit_Test_Submission.ipynb', 'repo_name': 'GilesStrong/Kaggle_HiggsML', 'size': '4251'}