<commit_before>'''
Main Integration class
Author: <NAME> <<EMAIL>[at]gmail.com>
Date: 16.04.2015
'''

import mushrooms_bfgs, randomForest, Sampling, PGCM, WilcoxonTest
from copy import deepcopy

if __name__ == '__main__':
    (train_X, train_y, test_X, test_y) = mushrooms_bfgs.initialize(float(80)/100)
    N = 5 # Number of Iterations at max can be equal to Number of features - 1
    for i in xrange(1, N + 1):
        # Maximum can remove all the features
        importanceOfRandomForest = randomForest.netFeatureImportance(train_X, train_y, 15)
        importanceOfSymbioticAlgorithm = Sampling.test_initialize()
        #~ print "Feature Importance by Random Forest ", importanceOfRandomForest
        #~ print "Feature Importance by Symbiotic Algorithm ", importanceOfSymbioticAlgorithm
        (train_X_Sym, test_X_Sym) = mushrooms_bfgs.featureRemoval(train_X, test_X, [importanceOfSymbioticAlgorithm[i - 1][0]])
        (train_X_For, test_X_For) = mushrooms_bfgs.featureRemoval(train_X, test_X, [importanceOfRandomForest[i - 1][0]])
        wilcoxonOut = WilcoxonTest.WilcoxonTest(train_X.ravel(), train_X_Sym.ravel() ,train_X_For.ravel())
        if wilcoxonOut == 1:
            newData_Tr = deepcopy(train_X_For)
            newData_Ts = deepcopy(test_X_For)
        elif wilcoxonOut == 0:
            newData_Tr = deepcopy(train_X_Sym)
            newData_Ts = deepcopy(test_X_Sym)
        else:
            break
        train_X = deepcopy(newData_Tr)
        test_X = deepcopy(newData_Ts)
        #~ print train_X[0]
<commit_msg>Added Main file with appropriate output<commit_after>'''
Main Integration class
Author: <NAME> <<EMAIL>[at]gmail.com>
Date: 16.04.2015
'''

import mushrooms_bfgs, randomForest, Sampling, PGCM, WilcoxonTest
from copy import deepcopy

if __name__ == '__main__':
    (train_X, train_y, test_X, test_y) = mushrooms_bfgs.initialize(float(80)/100)
    N = 22 # Number of Iterations at max can be equal to Number of features - 1
    NumberOfFeaturesRemoved = 0 
    FeaturesRemoved = []
    for i in xrange(1, N + 1):
        # Maximum can remove all the features
        importanceOfRandomForest = randomForest.netFeatureImportance(train_X, train_y, 15)
        importanceOfSymbioticAlgorithm = Sampling.test_initialize()
        #~ print "Feature Importance by Random Forest ", importanceOfRandomForest
        #~ print "Feature Importance by Symbiotic Algorithm ", importanceOfSymbioticAlgorithm
        if len(FeaturesRemoved) == 0 :
            (train_X_Sym, test_X_Sym) = mushrooms_bfgs.featureRemoval(train_X, test_X, [importanceOfSymbioticAlgorithm[i - 1][0]])
            (train_X_For, test_X_For) = mushrooms_bfgs.featureRemoval(train_X, test_X, [importanceOfRandomForest[i - 1][0]])
        else:
            trialFeatureSym = deepcopy(FeaturesRemoved)
            trialFeatureFor = deepcopy(FeaturesRemoved)
            trialFeatureSym.append(importanceOfSymbioticAlgorithm[i - 1][0])
            trialFeatureFor.append(importanceOfRandomForest[i - 1][0])
            print trialFeatureFor, trialFeatureSym
            (train_X_Sym, test_X_Sym) = mushrooms_bfgs.featureRemoval(train_X, test_X, trialFeatureSym)
            (train_X_For, test_X_For) = mushrooms_bfgs.featureRemoval(train_X, test_X, trialFeatureFor)
        wilcoxonOut = WilcoxonTest.WilcoxonTest(train_X.ravel(), train_X_Sym.ravel() ,train_X_For.ravel())
        if wilcoxonOut == 1:
            if importanceOfRandomForest[i - 1][0] in FeaturesRemoved:
                continue
            FeaturesRemoved.append(importanceOfRandomForest[i - 1][0])
            newData_Tr = deepcopy(train_X_For)
            newData_Ts = deepcopy(test_X_For)
        elif wilcoxonOut == 0:
            if importanceOfSymbioticAlgorithm[i - 1][0] in FeaturesRemoved:
                continue
            FeaturesRemoved.append(importanceOfSymbioticAlgorithm[i - 1][0])
            newData_Tr = deepcopy(train_X_Sym)
            newData_Ts = deepcopy(test_X_Sym)
        else:
            break
        #~ print newData_Tr[0]
        NumberOfFeaturesRemoved += 1
    print "Total Number of Features Removed is ", NumberOfFeaturesRemoved
    print "Features Removed are", FeaturesRemoved

{'fraction_non_alphanumeric': 0.09006211180124224, 'fraction_numerical': 0.016244624940277116, 'mean_word_length': 3.3209494324045408, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '541801', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1299, 'n_words': 284}