# Path: insults/util.py
# def make_sure_path_exists(path):
# def data_file(category, name=None):
# def data_directory(category):
# def log_file(name):
# def save_folds(folds):
# def saved_folds():
# def save_model( clf, location=SAVEFILE_LOCATION ):
# def load_model( location=SAVEFILE_LOCATION, model_filename=MODEL_FILENAME):
# def get_parser():
# def tied_rank(x):
# def auc(actual, posterior):
# def score():
# SAVEFILE_LOCATION = "models/stored/"
# MODEL_FILENAME = "insult_classifier.joblib.pkl"
# 
# Path: insults/pipeline.py
# def make_pipeline(options):
#     vect = ('vect', feature_extraction.text.CountVectorizer(lowercase=False,
#                                                             analyzer='char',
#                                                             ngram_range=(1, 5)))
#     tfidf = ('tfidf', feature_extraction.text.TfidfTransformer(sublinear_tf=True, norm='l2'))
#     clf = ("clf",InsultsSGDRegressor(alpha=options.sgd_alpha,
#                                      penalty=options.sgd_penalty,
#                                      learning_rate='constant',
#                                      eta0=options.sgd_eta0,
#                                      max_iter=options.sgd_max_iter,
#                                      n_iter_per_step=options.sgd_n_iter_per_step))
# 
#     return InsultsPipeline([vect, tfidf, clf])


import pandas
import numpy as np
import os
import itertools
import logging
import argparse
import sys
import concurrent.futures
from sklearn import model_selection
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from insults.util import (data_file,
                          data_directory,
                          log_file,
                          save_folds,
                          saved_folds,
                          argsets,
                          auc,
                          score,
                          save_model,
                          get_parser)
from insults.pipeline import make_pipeline
#
# training file + parameters -> folds -> chosen parameters + performance estimates
# training file + chosen parameters -> model
# labeled test file + model -> predictions -> score
# unlabeled test file + model -> predictions



NFOLDS=15


def get_estimates():
    estimate_file = data_file('Estimates','estimates.csv')
    if os.path.exists(estimate_file):
        v = pandas.read_table(estimate_file,sep=',')
        return zip(v.submission, v.estimate)
    else:
        return []


def save_estimates(se):
    estimate_file = data_file('Estimates', 'estimates.csv')
    submissions,estimates = zip(*se)
    pandas.DataFrame(dict(submission=submissions, estimate=estimates)).to_csv(estimate_file, index=False)


def make_full_training():
    df1 = pandas.read_table(data_file('Inputs','train.csv'),sep=',')
    df2 = pandas.read_table(data_file('Inputs','test_with_solutions.csv'),sep=',')
    df = pandas.concat([df1,df2])

# Path: insults/util.py
# def make_sure_path_exists(path):
# def data_file(category, name=None):
# def data_directory(category):
# def log_file(name):
# def save_folds(folds):
# def saved_folds():
# def save_model( clf, location=SAVEFILE_LOCATION ):
# def load_model( location=SAVEFILE_LOCATION, model_filename=MODEL_FILENAME):
# def get_parser():
# def tied_rank(x):
# def auc(actual, posterior):
# def score():
# SAVEFILE_LOCATION = "models/stored/"
# MODEL_FILENAME = "insult_classifier.joblib.pkl"
# 
# Path: insults/pipeline.py
# def make_pipeline(options):
#     vect = ('vect', feature_extraction.text.CountVectorizer(lowercase=False,
#                                                             analyzer='char',
#                                                             ngram_range=(1, 5)))
#     tfidf = ('tfidf', feature_extraction.text.TfidfTransformer(sublinear_tf=True, norm='l2'))
#     clf = ("clf",InsultsSGDRegressor(alpha=options.sgd_alpha,
#                                      penalty=options.sgd_penalty,
#                                      learning_rate='constant',
#                                      eta0=options.sgd_eta0,
#                                      max_iter=options.sgd_max_iter,
#                                      n_iter_per_step=options.sgd_n_iter_per_step))
# 
#     return InsultsPipeline([vect, tfidf, clf])


# Path: insults/train.py
import pandas
import numpy as np
import os
import itertools
import logging
import argparse
import sys
import concurrent.futures
from sklearn import model_selection
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from insults.util import (data_file,
                          data_directory,
                          log_file,
                          save_folds,
                          saved_folds,
                          argsets,
                          auc,
                          score,
                          save_model,
                          get_parser)
from insults.pipeline import make_pipeline

#
# training file + parameters -> folds -> chosen parameters + performance estimates
# training file + chosen parameters -> model
# labeled test file + model -> predictions -> score
# unlabeled test file + model -> predictions



NFOLDS=15


def get_estimates():
    estimate_file = data_file('Estimates','estimates.csv')
    if os.path.exists(estimate_file):
        v = pandas.read_table(estimate_file,sep=',')
        return zip(v.submission, v.estimate)
    else:
        return []


def save_estimates(se):
    estimate_file = data_file('Estimates', 'estimates.csv')
    submissions,estimates = zip(*se)
    pandas.DataFrame(dict(submission=submissions, estimate=estimates)).to_csv(estimate_file, index=False)


def make_full_training():
    df1 = pandas.read_table(data_file('Inputs','train.csv'),sep=',')
    df2 = pandas.read_table(data_file('Inputs','test_with_solutions.csv'),sep=',')
    df = pandas.concat([df1,df2])

    df.to_csv(data_file('Inputs','fulltrain.csv'),index=False)