# Path: automlk/metrics.py
# class Metric(object):
#     def __init__(self, name, function, problem_type, best_is_min, need_class=False, binary=False, average=False):
# def rmse(y_act, y_pred):
# def rmsle(y_act, y_pred):
# def gini(y_act, y_pred):
# def gini_normalized(y_act, y_pred):
# def evaluate_metric(y_act, y_pred, metric_name, n_classes):
# 
# Path: automlk/textset.py
# def get_textset_list():
#     """
#     get the list of all textsets
# 
#     :return: list of textsets objects or empty list if error (eg. redis or environment not set)
#     """
#     #try:
#     return [get_textset(textset_id) for textset_id in get_textset_ids()]
#     #except:
#     #    return []
# 
# Path: automlk/context.py
# def get_dataset_folder(dataset_id):
#     """
#     storage folder of the dataset
#     :param dataset_id: id of the dataset
#     :return: folder path (root of the structure)
#     """
#     return get_data_folder() + '/%s' % dataset_id
# 
# Path: automlk/folders.py
# def get_folder_list():
#     """
#     returns the list of folders
# 
#     :return:
#     """
#     if exists_key_store('folders:list'):
#         return get_key_store('folders:list')
#     else:
#         # initialize folder list with root (All)
#         set_key_store('folders:counter', 0)
#         rpush_key_store('folders:list', {'id': 0, 'parent': -1, 'name': 'All'})
#         return get_key_store('folders:list')
# 
# Path: automlk/xyset.py
# class XySet(object):
#     def __init__(self, X, y, X_train, y_train, X_test, y_test, X_submit,
#                  id_submit, cv_folds, y_eval_list, y_eval, i_eval, i_eval0):
#         """
#         this class stores the various data required for analysis and hyepr-optimisation
# 
#         :param X: X features for the complete dataset (excluding bechnmark/test and submit)
#         :param y: y for this complete set
#         :param X_train: X features for the train set after holdout
#         :param y_train: y for this set
#         :param X_test: X features for the holdout set or test set in benchmark
#         :param y_test: y for this set
#         :param X_submit: X features for the submit set (competition mode only)
#         :param id_submit: y for this set
#         :param cv_folds: cross validation folds
#         :param y_eval_list: list of folds, with indexes for each fold
#         :param y_eval: y for the eval set
#         :param i_eval: indexes for the eval set
#         :param i_eval: indexes for the 1st round of the eval set
#         """
#         self.X = X
#         self.y = y
#         self.X_train = X_train
#         self.y_train = y_train
#         self.X_test = X_test
#         self.y_test = y_test
#         self.X_submit = X_submit
#         self.id_submit = id_submit
#         self.cv_folds = cv_folds
#         self.y_eval_list = y_eval_list
#         self.y_eval = y_eval
#         self.i_eval = i_eval
#         self.i_eval0 = i_eval0


import logging
import pickle
import glob
import datetime
import shutil
import pandas as pd
import numpy as np
from .metrics import metric_map
from .store import *
from .textset import get_textset_list
from .context import get_dataset_folder
from .folders import get_folder_list
from .xyset import XySet
        self.n_unique_values = n_unique_values
        self.n_unique_ratio = n_unique_ratio
        self.first_unique_values = first_unique_values
        self.print_values = ', '.join([str(x) for x in self.first_unique_values])

        # initialize type
        if col_type != '':
            if col_type not in ['numerical', 'categorical', 'text', 'date']:
                raise ValueError('feature %s col type: %s should be numerical, date, categorical or text' % (name, col_type))
            self.col_type = col_type
        else:
            if raw_type.startswith('float'):
                self.col_type = 'numerical'
            elif raw_type.startswith('int'):
                self.col_type = 'numerical'
            elif raw_type.startswith('datetime'):
                self.col_type = 'date'
            else:
                # raw_type in ['str', 'object']:
                self.col_type = 'categorical'

            # additional rules
            if self.col_type == 'numerical' and self.n_unique_values < 10:
                self.col_type = 'categorical'
            if self.col_type == 'categorical' and self.n_unique_ratio > 0.5:
                self.col_type = 'text'
            if text_ref != '':
                if col_type != 'text':
                    raise ValueError(
                        'feature %s of type %s : text reference can only apply to text' % (name, col_type))

# Path: automlk/metrics.py
# class Metric(object):
#     def __init__(self, name, function, problem_type, best_is_min, need_class=False, binary=False, average=False):
# def rmse(y_act, y_pred):
# def rmsle(y_act, y_pred):
# def gini(y_act, y_pred):
# def gini_normalized(y_act, y_pred):
# def evaluate_metric(y_act, y_pred, metric_name, n_classes):
# 
# Path: automlk/textset.py
# def get_textset_list():
#     """
#     get the list of all textsets
# 
#     :return: list of textsets objects or empty list if error (eg. redis or environment not set)
#     """
#     #try:
#     return [get_textset(textset_id) for textset_id in get_textset_ids()]
#     #except:
#     #    return []
# 
# Path: automlk/context.py
# def get_dataset_folder(dataset_id):
#     """
#     storage folder of the dataset
#     :param dataset_id: id of the dataset
#     :return: folder path (root of the structure)
#     """
#     return get_data_folder() + '/%s' % dataset_id
# 
# Path: automlk/folders.py
# def get_folder_list():
#     """
#     returns the list of folders
# 
#     :return:
#     """
#     if exists_key_store('folders:list'):
#         return get_key_store('folders:list')
#     else:
#         # initialize folder list with root (All)
#         set_key_store('folders:counter', 0)
#         rpush_key_store('folders:list', {'id': 0, 'parent': -1, 'name': 'All'})
#         return get_key_store('folders:list')
# 
# Path: automlk/xyset.py
# class XySet(object):
#     def __init__(self, X, y, X_train, y_train, X_test, y_test, X_submit,
#                  id_submit, cv_folds, y_eval_list, y_eval, i_eval, i_eval0):
#         """
#         this class stores the various data required for analysis and hyepr-optimisation
# 
#         :param X: X features for the complete dataset (excluding bechnmark/test and submit)
#         :param y: y for this complete set
#         :param X_train: X features for the train set after holdout
#         :param y_train: y for this set
#         :param X_test: X features for the holdout set or test set in benchmark
#         :param y_test: y for this set
#         :param X_submit: X features for the submit set (competition mode only)
#         :param id_submit: y for this set
#         :param cv_folds: cross validation folds
#         :param y_eval_list: list of folds, with indexes for each fold
#         :param y_eval: y for the eval set
#         :param i_eval: indexes for the eval set
#         :param i_eval: indexes for the 1st round of the eval set
#         """
#         self.X = X
#         self.y = y
#         self.X_train = X_train
#         self.y_train = y_train
#         self.X_test = X_test
#         self.y_test = y_test
#         self.X_submit = X_submit
#         self.id_submit = id_submit
#         self.cv_folds = cv_folds
#         self.y_eval_list = y_eval_list
#         self.y_eval = y_eval
#         self.i_eval = i_eval
#         self.i_eval0 = i_eval0


# Path: automlk/dataset.py
import logging
import pickle
import glob
import datetime
import shutil
import pandas as pd
import numpy as np
from .metrics import metric_map
from .store import *
from .textset import get_textset_list
from .context import get_dataset_folder
from .folders import get_folder_list
from .xyset import XySet

        self.n_unique_values = n_unique_values
        self.n_unique_ratio = n_unique_ratio
        self.first_unique_values = first_unique_values
        self.print_values = ', '.join([str(x) for x in self.first_unique_values])

        # initialize type
        if col_type != '':
            if col_type not in ['numerical', 'categorical', 'text', 'date']:
                raise ValueError('feature %s col type: %s should be numerical, date, categorical or text' % (name, col_type))
            self.col_type = col_type
        else:
            if raw_type.startswith('float'):
                self.col_type = 'numerical'
            elif raw_type.startswith('int'):
                self.col_type = 'numerical'
            elif raw_type.startswith('datetime'):
                self.col_type = 'date'
            else:
                # raw_type in ['str', 'object']:
                self.col_type = 'categorical'

            # additional rules
            if self.col_type == 'numerical' and self.n_unique_values < 10:
                self.col_type = 'categorical'
            if self.col_type == 'categorical' and self.n_unique_ratio > 0.5:
                self.col_type = 'text'
            if text_ref != '':
                if col_type != 'text':
                    raise ValueError(
                        'feature %s of type %s : text reference can only apply to text' % (name, col_type))

                if text_ref not in get_textset_list():