# Path: insults/util.py
# def make_sure_path_exists(path):
# def data_file(category, name=None):
# def data_directory(category):
# def log_file(name):
# def save_folds(folds):
# def saved_folds():
# def save_model( clf, location=SAVEFILE_LOCATION ):
# def load_model( location=SAVEFILE_LOCATION, model_filename=MODEL_FILENAME):
# def get_parser():
# def tied_rank(x):
# def auc(actual, posterior):
# def score():
# SAVEFILE_LOCATION = "models/stored/"
# MODEL_FILENAME = "insult_classifier.joblib.pkl"
# 
# Path: insults/pipeline.py
# def make_pipeline(options):
#     vect = ('vect', feature_extraction.text.CountVectorizer(lowercase=False,
#                                                             analyzer='char',
#                                                             ngram_range=(1, 5)))
#     tfidf = ('tfidf', feature_extraction.text.TfidfTransformer(sublinear_tf=True, norm='l2'))
#     clf = ("clf",InsultsSGDRegressor(alpha=options.sgd_alpha,
#                                      penalty=options.sgd_penalty,
#                                      learning_rate='constant',
#                                      eta0=options.sgd_eta0,
#                                      max_iter=options.sgd_max_iter,
#                                      n_iter_per_step=options.sgd_n_iter_per_step))
# 
#     return InsultsPipeline([vect, tfidf, clf])


import pandas
import numpy as np
import os
import itertools
import logging
import argparse
import sys
import concurrent.futures
from sklearn import model_selection
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from insults.util import (data_file,
                          data_directory,
                          log_file,
                          save_folds,
                          saved_folds,
                          argsets,
                          auc,
                          score,
                          save_model,
                          get_parser)
from insults.pipeline import make_pipeline
    with ProcessPoolExecutor(max_workers=NUM_CORES) as executor:
        future_to_fold = dict([
                                (executor.submit(
                                                tune_one_fold,
                                                arguments,
                                                i,
                                                train_i,
                                                test_i
                                                ), i)
                                for i,(train_i,test_i)
                                in enumerate(boolean_folds)
                              ])

        df =  join_frames([
                            future.result()
                            for future
                            in concurrent.futures.as_completed(future_to_fold)
                          ])

    logging.info('tuning complete')

    return df


def predict(folds, arguments):
    """
    Train on training file, predict on test file.
    """
    logging.info("Starting predictions")


# Path: insults/util.py
# def make_sure_path_exists(path):
# def data_file(category, name=None):
# def data_directory(category):
# def log_file(name):
# def save_folds(folds):
# def saved_folds():
# def save_model( clf, location=SAVEFILE_LOCATION ):
# def load_model( location=SAVEFILE_LOCATION, model_filename=MODEL_FILENAME):
# def get_parser():
# def tied_rank(x):
# def auc(actual, posterior):
# def score():
# SAVEFILE_LOCATION = "models/stored/"
# MODEL_FILENAME = "insult_classifier.joblib.pkl"
# 
# Path: insults/pipeline.py
# def make_pipeline(options):
#     vect = ('vect', feature_extraction.text.CountVectorizer(lowercase=False,
#                                                             analyzer='char',
#                                                             ngram_range=(1, 5)))
#     tfidf = ('tfidf', feature_extraction.text.TfidfTransformer(sublinear_tf=True, norm='l2'))
#     clf = ("clf",InsultsSGDRegressor(alpha=options.sgd_alpha,
#                                      penalty=options.sgd_penalty,
#                                      learning_rate='constant',
#                                      eta0=options.sgd_eta0,
#                                      max_iter=options.sgd_max_iter,
#                                      n_iter_per_step=options.sgd_n_iter_per_step))
# 
#     return InsultsPipeline([vect, tfidf, clf])


# Path: insults/train.py
import pandas
import numpy as np
import os
import itertools
import logging
import argparse
import sys
import concurrent.futures
from sklearn import model_selection
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from insults.util import (data_file,
                          data_directory,
                          log_file,
                          save_folds,
                          saved_folds,
                          argsets,
                          auc,
                          score,
                          save_model,
                          get_parser)
from insults.pipeline import make_pipeline

    with ProcessPoolExecutor(max_workers=NUM_CORES) as executor:
        future_to_fold = dict([
                                (executor.submit(
                                                tune_one_fold,
                                                arguments,
                                                i,
                                                train_i,
                                                test_i
                                                ), i)
                                for i,(train_i,test_i)
                                in enumerate(boolean_folds)
                              ])

        df =  join_frames([
                            future.result()
                            for future
                            in concurrent.futures.as_completed(future_to_fold)
                          ])

    logging.info('tuning complete')

    return df


def predict(folds, arguments):
    """
    Train on training file, predict on test file.
    """
    logging.info("Starting predictions")


    clf = make_pipeline(arguments)