# Path: data_postp/matching.py
# def train_and_dump_classifier(df, dump_path=None, class_column="category", classifier=sklearn.linear_model.LogisticRegression(n_jobs=-1),
#                               n_components=500, train_split_ratio=0.8):
# 
#   # separate dataframe into test and train split
#   if train_split_ratio < 1.0:
#     msk = np.random.rand(len(df)) < train_split_ratio
#     test_df = df[~msk]
#     train_df = df[msk]
#   else: #no testing required -> make test_df dummy
#     train_df = df.copy()
#     test_df = df.copy()
# 
#   if n_components > 0:
#     pca = inter_class_pca(df, class_column=class_column, n_components=n_components)
#     X_train = pca.transform(df_col_to_matrix(train_df['hidden_repr']))
#     X_test = pca.transform(df_col_to_matrix(test_df['hidden_repr']))
# 
#   else:
#     X_train = df_col_to_matrix(train_df['hidden_repr'])
#     X_test = df_col_to_matrix(test_df['hidden_repr'])
# 
#   y_train = np.asarray(list(train_df[class_column]))
#   y_test = np.asarray(list(test_df[class_column]))
# 
# 
#   #fit model:
#   print("Training classifier")
#   classifier.fit(X_train, y_train)
# 
#   #calculate accuracy
#   if train_split_ratio < 1.0:
#     acc = classifier.score(X_test, y_test)
#     print("Accuracy:", acc)
#     top5_acc = top_n_accuracy(classifier, X_test, y_test, n=3)
#     print("Top5 Accuracy:", top5_acc)
#   else:
#     acc, top5_acc = None, None
# 
#   #dump classifier
#   if dump_path:
#     classifier_dump_path = os.path.join(os.path.dirname(dump_path), 'classifier.pickle')
#     with open(classifier_dump_path, 'wb') as f:
#       pickle.dump(classifier, f)
#     print("Dumped classifier to:", classifier_dump_path)
# 
#   return classifier, pca, acc, top5_acc
# 
# Path: data_postp/similarity_computations.py
# def df_col_to_matrix(panda_col):
#   """Converts a pandas dataframe column wherin each element in an ndarray into a 2D Matrix
#   :return ndarray (2D)
#   :param panda_col - panda Series wherin each element in an ndarray
#   """
#   panda_col = panda_col.map(lambda x: x.flatten())
#   return np.vstack(panda_col)


import numpy as np
import pandas as pd
import utils.io_handler as io_handler
import os, collections, scipy, itertools, multiprocessing, shutil
import scipy, pickle, json
import sklearn, sklearn.ensemble
import csv
from datetime import datetime
from pprint import pprint
from tensorflow.python.platform import flags
from data_postp.matching import train_and_dump_classifier
from data_postp.similarity_computations import df_col_to_matrix
  #train classifier on valid_df

  classifier, pca, _, _ = train_and_dump_classifier(valid_df, class_column="category",
                                                             classifier=sklearn.linear_model.LogisticRegression(n_jobs=-1),
                                                             n_components=n_components, train_split_ratio=0.8)

  test_df = pd.read_pickle(PICKLE_FILE_TEST)

  #PCA transform hidden_reps of test_df
  transformed_vectors_as_matrix = pca.transform(df_col_to_matrix(test_df['hidden_repr']))
  test_df['hidden_repr'] = np.split(transformed_vectors_as_matrix, transformed_vectors_as_matrix.shape[0])
  X_test = df_col_to_matrix(test_df['hidden_repr'])

  #predict labels
  Y = classifier.predict(X_test)

  #generate csv
  result_df = pd.DataFrame(Y, index=test_df['id'])
  result_df.to_csv(dump_path, sep=';')

def calculate_accuracy(valid_df, n_components=[200], n_folds=5,
                       classifier=sklearn.linear_model.LogisticRegression(n_jobs=-1), dump_file_name=None):
  n_components = [n_components] if type(n_components) is not list else n_components
  string_to_dump = ''

  for n_comp in n_components:
    print(type(n_comp))
    acc_array, top5_acc_array = [], []
    print("Training started with %i PCA components and %i folds" %(n_comp, n_folds))
    for i in range(n_folds):

# Path: data_postp/matching.py
# def train_and_dump_classifier(df, dump_path=None, class_column="category", classifier=sklearn.linear_model.LogisticRegression(n_jobs=-1),
#                               n_components=500, train_split_ratio=0.8):
# 
#   # separate dataframe into test and train split
#   if train_split_ratio < 1.0:
#     msk = np.random.rand(len(df)) < train_split_ratio
#     test_df = df[~msk]
#     train_df = df[msk]
#   else: #no testing required -> make test_df dummy
#     train_df = df.copy()
#     test_df = df.copy()
# 
#   if n_components > 0:
#     pca = inter_class_pca(df, class_column=class_column, n_components=n_components)
#     X_train = pca.transform(df_col_to_matrix(train_df['hidden_repr']))
#     X_test = pca.transform(df_col_to_matrix(test_df['hidden_repr']))
# 
#   else:
#     X_train = df_col_to_matrix(train_df['hidden_repr'])
#     X_test = df_col_to_matrix(test_df['hidden_repr'])
# 
#   y_train = np.asarray(list(train_df[class_column]))
#   y_test = np.asarray(list(test_df[class_column]))
# 
# 
#   #fit model:
#   print("Training classifier")
#   classifier.fit(X_train, y_train)
# 
#   #calculate accuracy
#   if train_split_ratio < 1.0:
#     acc = classifier.score(X_test, y_test)
#     print("Accuracy:", acc)
#     top5_acc = top_n_accuracy(classifier, X_test, y_test, n=3)
#     print("Top5 Accuracy:", top5_acc)
#   else:
#     acc, top5_acc = None, None
# 
#   #dump classifier
#   if dump_path:
#     classifier_dump_path = os.path.join(os.path.dirname(dump_path), 'classifier.pickle')
#     with open(classifier_dump_path, 'wb') as f:
#       pickle.dump(classifier, f)
#     print("Dumped classifier to:", classifier_dump_path)
# 
#   return classifier, pca, acc, top5_acc
# 
# Path: data_postp/similarity_computations.py
# def df_col_to_matrix(panda_col):
#   """Converts a pandas dataframe column wherin each element in an ndarray into a 2D Matrix
#   :return ndarray (2D)
#   :param panda_col - panda Series wherin each element in an ndarray
#   """
#   panda_col = panda_col.map(lambda x: x.flatten())
#   return np.vstack(panda_col)


# Path: data_postp/classification.py
import numpy as np
import pandas as pd
import utils.io_handler as io_handler
import os, collections, scipy, itertools, multiprocessing, shutil
import scipy, pickle, json
import sklearn, sklearn.ensemble
import csv
from datetime import datetime
from pprint import pprint
from tensorflow.python.platform import flags
from data_postp.matching import train_and_dump_classifier
from data_postp.similarity_computations import df_col_to_matrix

  #train classifier on valid_df

  classifier, pca, _, _ = train_and_dump_classifier(valid_df, class_column="category",
                                                             classifier=sklearn.linear_model.LogisticRegression(n_jobs=-1),
                                                             n_components=n_components, train_split_ratio=0.8)

  test_df = pd.read_pickle(PICKLE_FILE_TEST)

  #PCA transform hidden_reps of test_df
  transformed_vectors_as_matrix = pca.transform(df_col_to_matrix(test_df['hidden_repr']))
  test_df['hidden_repr'] = np.split(transformed_vectors_as_matrix, transformed_vectors_as_matrix.shape[0])
  X_test = df_col_to_matrix(test_df['hidden_repr'])

  #predict labels
  Y = classifier.predict(X_test)

  #generate csv
  result_df = pd.DataFrame(Y, index=test_df['id'])
  result_df.to_csv(dump_path, sep=';')

def calculate_accuracy(valid_df, n_components=[200], n_folds=5,
                       classifier=sklearn.linear_model.LogisticRegression(n_jobs=-1), dump_file_name=None):
  n_components = [n_components] if type(n_components) is not list else n_components
  string_to_dump = ''

  for n_comp in n_components:
    print(type(n_comp))
    acc_array, top5_acc_array = [], []
    print("Training started with %i PCA components and %i folds" %(n_comp, n_folds))
    for i in range(n_folds):

      _, _, acc, top5_acc = train_and_dump_classifier(valid_df, class_column="category", classifier=classifier,