# Path: automlk/monitor.py
# def heart_beep(module, msg, index=1, gpu=False):
#     """
#     send heart beep as module
# 
#     :param module: controller / worker
#     :param msg: status message to send to the web app
#     :param index: index of the worker
#     :param gpu: gpu capability
#     :return:
#     """
#     global __cpu_pct
#     if module == 'controller':
#         __cpu_pct = psutil.cpu_percent(interval=0.1)
# 
#     # detect IP and DNS name
#     t = datetime.datetime.now()
#     id = socket.gethostname() + '_' + str(index)
#     msg_beep = {'module': module,
#                 'host_name': id,
#                 'cpu_count': psutil.cpu_count(logical=False),
#                 'cpu_log_count': psutil.cpu_count(logical=True),
#                 'cpu_pct': __cpu_pct,
#                 'memory': psutil.virtual_memory().total/1073741824,
#                 'memory_pct': psutil.virtual_memory().percent,
#                 'version': __version__,
#                 'time': {'year': t.year, 'month': t.month, 'day': t.day,
#                          'hour': t.hour, 'minute': t.minute, 'second': t.second},
#                 'msg': msg
#                 }
# 
#     # update list
#     sadd_key_store('monitor:%s' % module, id)
# 
#     # save msg in store
#     set_key_store('monitor:%s:%s' % (module, id), msg_beep)
# 
# Path: automlk/dataset.py
# def get_dataset_list(include_results=False):
#     """
#     get the list of all datasets
# 
#     :param include_results: flag to determine if the status are also retrieved (default = False)
#     :return: list of datasets objects or empty list if error (eg. redis or environment not set)
#     """
#     # try:
#     return [get_dataset(dataset_id, include_results) for dataset_id in get_dataset_ids()]
#     # except:
#     #    return []
# 
# def get_dataset(dataset_id, include_results=False):
#     """
#     get the descriptive data of a dataset
# 
#     :param dataset_id: id of the dataset
#     :param include_results: if need to extract results also
#     :return: dataset object
#     """
#     dt = make_dataset(get_key_store('dataset:%s' % dataset_id))
#     dt.status = get_key_store('dataset:%s:status' % dataset_id)
# 
#     # add counters and results
#     dt.grapher = get_key_store('dataset:%s:grapher' % dataset_id)
#     dt.round_counter = get_counter_store('dataset:%s:round_counter' % dataset_id)
# 
#     if include_results:
#         dt.results = get_key_store('dataset:%s:results' % dataset_id)
# 
#     return dt
# 
# Path: automlk/specific.py
# def get_feature_engineering(dataset_id):
#     """
#     retrieves the feature engineering content of a dataset
# 
#     :param dataset_id: id of the dataset
#     :return: content: text of the python function (without def and return)
#     """
#     filename = get_dataset_folder(dataset_id) + '/data/fe_%s.txt' % dataset_id
#     if os.path.exists(filename):
#         with open(filename, 'r') as f:
#             return "".join(f.readlines())
#     else:
#         return ''
# 
# def apply_feature_engineering(dataset_id, X=None):
#     """
#     executes the feature engineering definition
# 
#     :param dataset_id: dataset id
#     :return:
#     """
#     global specific_feature_engineering
#     exec_feature_engineering(get_feature_engineering(dataset_id))
#     if isinstance(X, pd.DataFrame):
#         return specific_feature_engineering(X)
#     else:
#         return None


import time
import pickle
from .store import *
from .graphs import *
from .monitor import heart_beep
from .dataset import get_dataset_list, get_dataset
from .specific import get_feature_engineering, apply_feature_engineering


def grapher_loop():
    """
    periodically pool the grapher queue for a job

    :return: 
    """
    while True:
        heart_beep('grapher', '')
        # check the list of datasets
        for dt in get_dataset_list():
            if dt.status != 'created' and not dt.grapher:
                heart_beep('grapher', {'dataset_id': dt.dataset_id, 'dataset_name': dt.name})
                logging.info('grapher on dataset: %s' % dt.dataset_id)
                create_graph_data(dt.dataset_id)
        time.sleep(60)


def create_graph_data(dataset_id):
    """
    creates the graphs for each column feature of the dataset

    :param dataset_id: dataset id
    :return:
    """
    dataset = get_dataset(dataset_id)
    df = dataset.get_data()

    # apply feature engineering (if any)

# Path: automlk/monitor.py
# def heart_beep(module, msg, index=1, gpu=False):
#     """
#     send heart beep as module
# 
#     :param module: controller / worker
#     :param msg: status message to send to the web app
#     :param index: index of the worker
#     :param gpu: gpu capability
#     :return:
#     """
#     global __cpu_pct
#     if module == 'controller':
#         __cpu_pct = psutil.cpu_percent(interval=0.1)
# 
#     # detect IP and DNS name
#     t = datetime.datetime.now()
#     id = socket.gethostname() + '_' + str(index)
#     msg_beep = {'module': module,
#                 'host_name': id,
#                 'cpu_count': psutil.cpu_count(logical=False),
#                 'cpu_log_count': psutil.cpu_count(logical=True),
#                 'cpu_pct': __cpu_pct,
#                 'memory': psutil.virtual_memory().total/1073741824,
#                 'memory_pct': psutil.virtual_memory().percent,
#                 'version': __version__,
#                 'time': {'year': t.year, 'month': t.month, 'day': t.day,
#                          'hour': t.hour, 'minute': t.minute, 'second': t.second},
#                 'msg': msg
#                 }
# 
#     # update list
#     sadd_key_store('monitor:%s' % module, id)
# 
#     # save msg in store
#     set_key_store('monitor:%s:%s' % (module, id), msg_beep)
# 
# Path: automlk/dataset.py
# def get_dataset_list(include_results=False):
#     """
#     get the list of all datasets
# 
#     :param include_results: flag to determine if the status are also retrieved (default = False)
#     :return: list of datasets objects or empty list if error (eg. redis or environment not set)
#     """
#     # try:
#     return [get_dataset(dataset_id, include_results) for dataset_id in get_dataset_ids()]
#     # except:
#     #    return []
# 
# def get_dataset(dataset_id, include_results=False):
#     """
#     get the descriptive data of a dataset
# 
#     :param dataset_id: id of the dataset
#     :param include_results: if need to extract results also
#     :return: dataset object
#     """
#     dt = make_dataset(get_key_store('dataset:%s' % dataset_id))
#     dt.status = get_key_store('dataset:%s:status' % dataset_id)
# 
#     # add counters and results
#     dt.grapher = get_key_store('dataset:%s:grapher' % dataset_id)
#     dt.round_counter = get_counter_store('dataset:%s:round_counter' % dataset_id)
# 
#     if include_results:
#         dt.results = get_key_store('dataset:%s:results' % dataset_id)
# 
#     return dt
# 
# Path: automlk/specific.py
# def get_feature_engineering(dataset_id):
#     """
#     retrieves the feature engineering content of a dataset
# 
#     :param dataset_id: id of the dataset
#     :return: content: text of the python function (without def and return)
#     """
#     filename = get_dataset_folder(dataset_id) + '/data/fe_%s.txt' % dataset_id
#     if os.path.exists(filename):
#         with open(filename, 'r') as f:
#             return "".join(f.readlines())
#     else:
#         return ''
# 
# def apply_feature_engineering(dataset_id, X=None):
#     """
#     executes the feature engineering definition
# 
#     :param dataset_id: dataset id
#     :return:
#     """
#     global specific_feature_engineering
#     exec_feature_engineering(get_feature_engineering(dataset_id))
#     if isinstance(X, pd.DataFrame):
#         return specific_feature_engineering(X)
#     else:
#         return None


# Path: automlk/grapher.py
import time
import pickle
from .store import *
from .graphs import *
from .monitor import heart_beep
from .dataset import get_dataset_list, get_dataset
from .specific import get_feature_engineering, apply_feature_engineering



def grapher_loop():
    """
    periodically pool the grapher queue for a job

    :return: 
    """
    while True:
        heart_beep('grapher', '')
        # check the list of datasets
        for dt in get_dataset_list():
            if dt.status != 'created' and not dt.grapher:
                heart_beep('grapher', {'dataset_id': dt.dataset_id, 'dataset_name': dt.name})
                logging.info('grapher on dataset: %s' % dt.dataset_id)
                create_graph_data(dt.dataset_id)
        time.sleep(60)


def create_graph_data(dataset_id):
    """
    creates the graphs for each column feature of the dataset

    :param dataset_id: dataset id
    :return:
    """
    dataset = get_dataset(dataset_id)
    df = dataset.get_data()

    # apply feature engineering (if any)

    fe = get_feature_engineering(dataset.dataset_id)