<commit_before>"""Train the models
"""
from keras.callbacks import EarlyStopping, History, TensorBoard
from keras.models import load_model
import hyperopt
from hyperopt.utils import coarse_utcnow
from hyperopt.mongoexp import MongoTrials
import concise.eval_metrics as ce
from concise.utils.helper import write_json, merge_dicts
from concise.utils.model_data import (subset, split_train_test_idx, split_KFold_idx)
from datetime import datetime, timedelta
from uuid import uuid4
from hyperopt import STATUS_OK
import numpy as np
import pandas as pd
from copy import deepcopy
import os
import glob
import pprint
import logging
import matplotlib.pyplot as plt


logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s')
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# TODO - have a system-wide config for this
DEFAULT_IP = "ouga03"
DEFAULT_SAVE_DIR = "/s/project/deepcis/hyperopt/"

# TODO - think about the workflow
# def oof_predictions(fn, params, epochs, model_name, cache_model=True, save_model=True):
#     model_dir = fn.save_dir_exp + "/eval_models/"
#     model_path = model_dir + "/{0}.h5".format(model_name)
#     eval_metrics_path = model_dir + "/{0}_eval_metrics.json".format(model_name)

#     train, test = get_data(fn.data_fn, params)

#     if cache_model and os.path.isfile(model_path):
#         model = load_model(model_path)
#     else:
#         model = get_model(fn.model_fn, train, params)
#         model.train(train[0], train[1],
#                     batch_size=params["fit"]["batch_size"],
#                     epochs=epochs)

#     y_pred = model.predict(test[0])
#     eval_metrics = eval_model(model, test, fn.add_eval_metrics)

#     if save_model:
#         os.makedirs(model_dir, exist_ok=True)
#         # TODO - how to name things
#         model.save(model_path)

#     #write_json(eval_metrics, eval_metrics_path)

#     return eval_metrics, {"y_pred": y_pred, "y_true": test[1]}


def test_fn(fn, hyper_params, n_train=1000, tmp_dir="/tmp/concise_hyopt_test/"):
    """Test the correctness of the function before executing on large scale
    1. Run without error
    2. Correct save/load model to disk

    # Arguments

        n_train: int, number of training points
    """
    def wrap_data_fn(data_fn, n_train=100):
        def new_data_fn(*args, **kwargs):
            train, test = data_fn(*args, **kwargs)
            train = subset(train, np.arange(n_train))
            return train, test
        return new_data_fn
    start_time = datetime.now()
    fn = deepcopy(fn)
    hyper_params = deepcopy(hyper_params)
    fn.save_dir = tmp_dir
    fn.save_model = True
    fn.data_fn = wrap_data_fn(fn.data_fn, n_train)

    # sample from hyper_params
    param = hyperopt.pyll.stochastic.sample(hyper_params)
    # overwrite the number of epochs
    if param.get("fit") is None:
        param["fit"] = {}
    param["fit"]["epochs"] = 1

    # correct execution
    res = fn(param)
    print("Returned value:")
    pprint.pprint(res)
    assert res["status"] == STATUS_OK

    # correct model loading
    model_path = max(glob.iglob(fn.save_dir_exp + '/train_models/*.h5'),
                     key=os.path.getctime)
    assert datetime.fromtimestamp(os.path.getctime(model_path)) > start_time
    load_model(model_path)


class CMongoTrials(MongoTrials):<commit_msg>fix: if n_train larger than number of datapoints, truncate<commit_after>"""Train the models
"""
from keras.callbacks import EarlyStopping, History, TensorBoard
from keras.models import load_model
import hyperopt
from hyperopt.utils import coarse_utcnow
from hyperopt.mongoexp import MongoTrials
import concise.eval_metrics as ce
from concise.utils.helper import write_json, merge_dicts
from concise.utils.model_data import (subset, split_train_test_idx, split_KFold_idx)
from datetime import datetime, timedelta
from uuid import uuid4
from hyperopt import STATUS_OK
import numpy as np
import pandas as pd
from copy import deepcopy
import os
import glob
import pprint
import logging
import matplotlib.pyplot as plt


logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s')
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# TODO - have a system-wide config for this
DEFAULT_IP = "ouga03"
DEFAULT_SAVE_DIR = "/s/project/deepcis/hyperopt/"

# TODO - think about the workflow
# def oof_predictions(fn, params, epochs, model_name, cache_model=True, save_model=True):
#     model_dir = fn.save_dir_exp + "/eval_models/"
#     model_path = model_dir + "/{0}.h5".format(model_name)
#     eval_metrics_path = model_dir + "/{0}_eval_metrics.json".format(model_name)

#     train, test = get_data(fn.data_fn, params)

#     if cache_model and os.path.isfile(model_path):
#         model = load_model(model_path)
#     else:
#         model = get_model(fn.model_fn, train, params)
#         model.train(train[0], train[1],
#                     batch_size=params["fit"]["batch_size"],
#                     epochs=epochs)

#     y_pred = model.predict(test[0])
#     eval_metrics = eval_model(model, test, fn.add_eval_metrics)

#     if save_model:
#         os.makedirs(model_dir, exist_ok=True)
#         # TODO - how to name things
#         model.save(model_path)

#     #write_json(eval_metrics, eval_metrics_path)

#     return eval_metrics, {"y_pred": y_pred, "y_true": test[1]}


def test_fn(fn, hyper_params, n_train=1000, tmp_dir="/tmp/concise_hyopt_test/"):
    """Test the correctness of the function before executing on large scale
    1. Run without error
    2. Correct save/load model to disk

    # Arguments
        fn: CompileFN instance
        hyper_params: pyll graph of hyper-parameters - as later provided to `hyperopt.fmin`
        n_train: int, number of training points
        tmp_dir: Temporary path where to write the trained model.
    """
    def wrap_data_fn(data_fn, n_train=100):
        def new_data_fn(*args, **kwargs):
            data = data_fn(*args, **kwargs)
            train = data[0]
            train = subset(train, idx=np.arange(min(n_train, train[1].shape[0])))
            return train, data[1]
        return new_data_fn
    start_time = datetime.now()
    fn = deepcopy(fn)
    hyper_params = deepcopy(hyper_params)
    fn.save_dir = tmp_dir
    fn.save_model = True
    fn.data_fn = wrap_data_fn(fn.data_fn, n_train)

    # sample from hyper_params
    param = hyperopt.pyll.stochastic.sample(hyper_params)
    # overwrite the number of epochs
    if param.get("fit") is None:
        param["fit"] = {}
    param["fit"]["epochs"] = 1

    # correct execution
    res = fn(param)
    print("Returned value:")
    pprint.pprint(res)
    assert res["status"] == STATUS_OK

    # correct model loading
    model_path = max(glob.iglob(fn.save_dir_exp + '/train_models/*.h5'),
                     key=os.path.getctime)
    assert datetime.fromtimestamp(os.path.getctime(model_path)) > start_time
    load_model(model_path)


class CMongoTrials(MongoTrials):
{'fraction_non_alphanumeric': 0.10703495899870523, 'fraction_numerical': 0.006617752841317796, 'mean_word_length': 3.4996763754045306, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6091615', 'n_tokens_mistral': 2327, 'n_tokens_neox': 2271, 'n_words': 551}