<commit_before>#Libraries from scikit-learn
from sklearn import preprocessing, cross_validation, metrics, tree, decomposition
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier, AdaBoostClassifier
from sklearn.linear_model import LogisticRegression, Perceptron, SGDClassifier, OrthogonalMatchingPursuit, RandomizedLogisticRegression
from sklearn.neighbors.nearest_centroid import NearestCentroid
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import train_test_split
from sklearn_evaluation.metrics import precision_at
from sklearn.grid_search import ParameterGrid, GridSearchCV
from sklearn.metrics import *
from sklearn.preprocessing import StandardScaler
import sklearn.feature_selection
import sklearn.pipeline

#All other libraries
import argparse, sys, pickle, random, time, json, datetime, itertools, csv
import matplotlib
import pandas as pd
import numpy as np
import pylab as pl
from scipy import optimize
import matplotlib.pyplot as plt
matplotlib.style.use('ggplot')
from pydoc import locate
from argparse import ArgumentParser

#Own modules
import evaluation
from utils import *

'''
GLOBAL VARIABLES
'''
#create timestamp
TS = time.time()
TIMESTAMP = datetime.datetime.fromtimestamp(TS).strftime('%Y-%m-%d_%H_%M_%S')


TRAIN_SPLITS_WINDOW =[[0,1],[1,2],[2,3]]
TEST_SPLITS_WINDOW = [2,3,4]
TRAIN_SPLITS_TACK =[[0,1],[0,1,2],[0,1,2,3]]
TEST_SPLITS_TACK = [2,3,4]
TO_DROP = ['crid', 'officer_id', 'agesqrd', 'index', 'Unnamed: 0', '']
FILL_WITH_MEAN = ['officers_age', 'transit_time', 'car_time']

#estimators
CLFS = {
        'GB': GradientBoostingClassifier(
                    learning_rate=0.05,
                    subsample=0.5,
                    max_depth=6,
                    n_estimators=10)
        }

GRID = {
        'GB': {
                'n_estimators': [1,10,100,1000],
                'learning_rate' : [0.001,0.01,0.05,0.1,0.5],
                'subsample' : [0.1,0.5,1.0],
                'max_depth': [1,3,5,10,20,50,100]
                }
       }


if __name__ == "__main__":
    '''
    Argument Parsing
    '''
    parser = argparse.ArgumentParser()
    parser.add_argument('csv', help='csv filename')
    parser.add_argument('label', help='label of dependent variable')
    parser.add_argument('to_try', nargs='+', default=['DT'], help='list of model abbreviations')
    parser.add_argument('--tack_one_on', action="store_true", default=False, help='use tack one on window validation')
    parser.add_argument('--demographic', action="store_true", default=False, help='using demographic info')
    args = parser.parse_args()

    '''
    Labels for Documenting
    '''
    label = args.label #y, predicted variable
    dem_label = 'with_demo' if args.demographic else 'non-demo'
    stage = 'stage-1'

    if args.tack_one_on:
        train_split, test_split = TRAIN_SPLITS_TACK, TEST_SPLITS_TACK
    else:
        train_split, test_split = TRAIN_SPLITS_WINDOW, TEST_SPLITS_WINDOW

    to_try = args.to_try
    df = pd.read_csv(args.csv)
    if args.demographic:
        FILL_WITH_MEAN.append('complainant_age')

    '''
    Trying Models
    '''
    for model_name in to_try:
        print(model_name)
        '''
        Processing and Imputation
        '''
        this_df = df
        for col in TO_DROP:
            try:
                this_df.drop(col, axis=1, inplace = True)
            except:
                continue

        chunks = temporal_split_data(this_df)

        #IMPUTE EACH TEMPORAL CHUNK
        for chunk in chunks:
            for col in chunk.columns:
                try:
                    if col in FILL_WITH_MEAN:
                        mean = round(df[col].mean())<commit_msg>fixed mean<commit_after>#Libraries from scikit-learn
from sklearn import preprocessing, cross_validation, metrics, tree, decomposition
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier, AdaBoostClassifier
from sklearn.linear_model import LogisticRegression, Perceptron, SGDClassifier, OrthogonalMatchingPursuit, RandomizedLogisticRegression
from sklearn.neighbors.nearest_centroid import NearestCentroid
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import train_test_split
from sklearn_evaluation.metrics import precision_at
from sklearn.grid_search import ParameterGrid, GridSearchCV
from sklearn.metrics import *
from sklearn.preprocessing import StandardScaler
import sklearn.feature_selection
import sklearn.pipeline

#All other libraries
import argparse, sys, pickle, random, time, json, datetime, itertools, csv
import matplotlib
import pandas as pd
import numpy as np
import pylab as pl
from scipy import optimize
import matplotlib.pyplot as plt
matplotlib.style.use('ggplot')
from pydoc import locate
from argparse import ArgumentParser

#Own modules
import evaluation
from utils import *

'''
GLOBAL VARIABLES
'''
#create timestamp
TS = time.time()
TIMESTAMP = datetime.datetime.fromtimestamp(TS).strftime('%Y-%m-%d_%H_%M_%S')


TRAIN_SPLITS_WINDOW =[[0,1],[1,2],[2,3]]
TEST_SPLITS_WINDOW = [2,3,4]
TRAIN_SPLITS_TACK =[[0,1],[0,1,2],[0,1,2,3]]
TEST_SPLITS_TACK = [2,3,4]
TO_DROP = ['crid', 'officer_id', 'agesqrd', 'index', 'Unnamed: 0', '']
FILL_WITH_MEAN = ['officers_age', 'transit_time', 'car_time']

#estimators
CLFS = {
        'GB': GradientBoostingClassifier(
                    learning_rate=0.05,
                    subsample=0.5,
                    max_depth=6,
                    n_estimators=10)
        }


if __name__ == "__main__":
    '''
    Argument Parsing
    '''
    parser = argparse.ArgumentParser()
    parser.add_argument('csv', help='csv filename')
    parser.add_argument('label', help='label of dependent variable')
    parser.add_argument('to_try', nargs='+', default=['DT'], help='list of model abbreviations')
    parser.add_argument('--tack_one_on', action="store_true", default=False, help='use tack one on window validation')
    parser.add_argument('--demographic', action="store_true", default=False, help='using demographic info')
    args = parser.parse_args()

    '''
    Labels for Documenting
    '''
    label = args.label #y, predicted variable
    dem_label = 'with_demo' if args.demographic else 'non-demo'
    stage = 'stage-1'

    if args.tack_one_on:
        train_split, test_split = TRAIN_SPLITS_TACK, TEST_SPLITS_TACK
    else:
        train_split, test_split = TRAIN_SPLITS_WINDOW, TEST_SPLITS_WINDOW

    to_try = args.to_try
    df = pd.read_csv(args.csv)
    if args.demographic:
        FILL_WITH_MEAN.append('complainant_age')

    '''
    Trying Models
    '''
    for model_name in to_try:
        print(model_name)
        '''
        Processing and Imputation
        '''
        this_df = df
        for col in TO_DROP:
            try:
                this_df.drop(col, axis=1, inplace = True)
            except:
                continue

        chunks = temporal_split_data(this_df)

        #IMPUTE EACH TEMPORAL CHUNK
        for chunk in chunks:
            for col in chunk.columns:
                try:
                    if col in FILL_WITH_MEAN:
                        mean = round(chunk[col].mean())
{'fraction_non_alphanumeric': 0.10346740638002774, 'fraction_numerical': 0.014701803051317614, 'mean_word_length': 3.294818344252531, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '334847', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2296, 'n_words': 532}