<commit_before>import numpy as np
import matplotlib.pyplot as plt

# Basic script to run network state classfication

from loadSeizureData  import LoadSeizureData
from classifierTester import ClassifierTester
from relabeling_functions import relabel, reorder
from visualisation import plots

from extractors.basicFeatures    import BasicFeatures
from extractors.freqfeatures     import FreqFeatures
from extractors.waveletfeatures import WaveletFeatures

from classifiers.randomForestClassifier import RandomForest
from classifiers.svm import SupportVecClf
from classifiers.neighbors import KNeighbors

dirpath = '/Users/Jonathan/Documents/PhD /Seizure_related/Network_states/VMData/Classified'
dataobj = LoadSeizureData(dirpath)
dataobj.load_data()
dataobj = relabel(dataobj)
dataobj = reorder(dataobj)
basicStatsExtractor = BasicFeatures()
wavelets = WaveletFeatures()
fourier = FreqFeatures()
#dataobj.extract_feature_array([basicStatsExtractor])
dataobj.extract_feature_array([wavelets, basicStatsExtractor, fourier])

print dataobj.features.shape
rf = RandomForest(no_trees = 100)
classtester = ClassifierTester(dataobj.features,np.ravel(dataobj.label_colarray), training_test_split = 80)
(score, predictedlabelsprobs, reallabels) = classtester.test_classifier(rf)
print 'training a random forest classifier!'
print score, 'percent correct!'

print 'training a support vector classifier'
svcclf = SupportVecClf(k_type = 'rbf')
(score, predictedlabelsprobs, reallabels) = classtester.test_classifier(svcclf)
print score, 'percent correct!'

print 'training a K nearest neighbors classifier'
knn_clf = KNeighbors(15)
(score, predictedlabelsprobs, reallabels) = classtester.test_classifier(knn_clf)
print score, 'percent correct!'

plots.radviz(dataobj)
#plots.scatter_matrix(dataobj)
#raw_input()<commit_msg>added frequency features<commit_after>import numpy as np
import matplotlib.pyplot as plt

# Basic script to run network state classfication

from loadSeizureData  import LoadSeizureData
from classifierTester import ClassifierTester
from relabeling_functions import relabel, reorder
from visualisation import plots

from extractors.basicFeatures    import BasicFeatures
from extractors.freqfeatures     import FreqFeatures
from extractors.waveletfeatures import WaveletFeatures

from classifiers.randomForestClassifier import RandomForest
from classifiers.svm import SupportVecClf
from classifiers.neighbors import KNeighbors

dirpath = '/Users/Jonathan/Documents/PhD /Seizure_related/Network_states/VMData/Classified'
dataobj = LoadSeizureData(dirpath)
dataobj.load_data()
dataobj = relabel(dataobj)
dataobj = reorder(dataobj)
basicStatsExtractor = BasicFeatures()
wavelets = WaveletFeatures()
fourier = FreqFeatures()
#dataobj.extract_feature_array([basicStatsExtractor])
dataobj.extract_feature_array([wavelets, basicStatsExtractor, fourier])


from sklearn.decomposition import PCA
pca = PCA(n_components=5)
pca.whiten =True
dataobj.features = pca.fit_transform(dataobj.features)


print dataobj.features.shape
rf = RandomForest(no_trees = 100)
classtester = ClassifierTester(dataobj.features,np.ravel(dataobj.label_colarray), training_test_split = 80)
(score, predictedlabelsprobs, reallabels) = classtester.test_classifier(rf)
print 'training a random forest classifier!'
print score, 'percent correct!'

print 'training a support vector classifier'
svcclf = SupportVecClf(k_type = 'rbf')
(score, predictedlabelsprobs, reallabels) = classtester.test_classifier(svcclf)
print score, 'percent correct!'

print 'training a K nearest neighbors classifier'
knn_clf = KNeighbors(15)
(score, predictedlabelsprobs, reallabels) = classtester.test_classifier(knn_clf)
print score, 'percent correct!'

plots.radviz(dataobj)
#plots.scatter_matrix(dataobj)
#raw_input()
{'fraction_non_alphanumeric': 0.07555673382820785, 'fraction_numerical': 0.003976670201484623, 'mean_word_length': 8.004773269689737, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2051901', 'n_tokens_mistral': 1198, 'n_tokens_neox': 1146, 'n_words': 283}