<commit_before>#!/usr/bin/env python2

from multiprocessing import cpu_count
import os

from sklearn import svm
from sklearn.cross_validation import KFold
from sklearn.cross_validation import train_test_split
from sklearn.externals import joblib
from sklearn.grid_search import GridSearchCV
from sklearn.preprocessing import StandardScaler

from numpy import logspace

from FeatMat import *

class Trainer:
    clf = None
    svm = None

    def __init__(self):
        self.svm = svm.SVC(kernel='linear', shrinking=True, verbose=False)
        params = {
            'C': np.linspace(1e-5, 1e-3, num=100), # Range of C values
        }
        self.clf = GridSearchCV(self.svm, params,
            cv      = 5,           # 5-fold CV
            n_jobs  = cpu_count(), # Parallelize over CPUs
            verbose = 2,
        )

    def train(self, featMat):
        # Preprocess
        scaler = StandardScaler()
        scaler.fit_transform(featMat.X, featMat.y)

        # Save preprocess output
        joblib.dump(scaler, 'preprocess.out')

        # Perform CV
        print('Running SVM trainer on %d rows of data with %d features.' % featMat.X.shape)
        self.clf.fit(featMat.X, featMat.y)

        # Save CV output
        print(self.clf.best_estimator_)
        joblib.dump(self.clf.best_estimator_, 'cv.out')


def main():
    # Go to script's directory
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    # Train on Wookie dataset, evaluate with Michael dataset
    featMat = FeatMat()
    featMat.addFolder('../datasets/Wookie')

    trainer = Trainer()
    trainer.train(featMat)

if __name__ == '__main__':
    main()

<commit_msg>Fix suitability training params<commit_after>#!/usr/bin/env python2

from multiprocessing import cpu_count
import os

from sklearn import svm
from sklearn.cross_validation import KFold
from sklearn.cross_validation import train_test_split
from sklearn.externals import joblib
from sklearn.grid_search import GridSearchCV
from sklearn.preprocessing import StandardScaler

from numpy import logspace

from FeatMat import *

class Trainer:
    clf = None
    svm = None

    def __init__(self):
        self.svm = svm.SVC(kernel='linear', shrinking=True, verbose=False)
        params = {
            'C': np.logspace(-4, -3, num=50), # Range of C values
        }
        self.clf = GridSearchCV(self.svm, params,
            cv      = 5,           # 5-fold CV
            n_jobs  = cpu_count(), # Parallelize over CPUs
            verbose = 2,
        )

    def train(self, featMat):
        # Preprocess
        scaler = StandardScaler()
        scaler.fit_transform(featMat.X, featMat.y)

        # Save preprocess output
        joblib.dump(scaler, 'preprocess.out')

        # Perform CV
        print('Running SVM trainer on %d rows of data with %d features.' % featMat.X.shape)
        self.clf.fit(featMat.X, featMat.y)

        # Save CV output
        print(self.clf.best_estimator_)
        joblib.dump(self.clf.best_estimator_, 'cv.out')


def main():
    # Go to script's directory
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    # Train on Wookie dataset, evaluate with Michael dataset
    featMat = FeatMat()
    featMat.addFolder('../datasets/Wookie')

    trainer = Trainer()
    trainer.train(featMat)

if __name__ == '__main__':
    main()


{'fraction_non_alphanumeric': 0.09775112443778111, 'fraction_numerical': 0.005697151424287856, 'mean_word_length': 2.6025917926565874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5658336', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1056, 'n_words': 271}