<commit_before>import ConfigParser
import json
import datetime
import os
import shutil
import numpy as np
import ast

# TODO: convert output to dictionary
def load_configuration(cfg_fn):
    config = ConfigParser.ConfigParser()

    config.read(cfg_fn)

    if not os.path.exists("experiment_results"):
        os.makedirs("experiment_results")

    try:
        set_name = "%s" % (config.get("exp_details", "setname"))
    except:
        set_name = "anonymous_set"

    if not os.path.exists("experiment_results/" + set_name):
        os.makedirs("experiment_results/" + set_name)

    dir_name = "experiment_results/%s/%s__%s/" % (set_name, config.get("exp_details", "name"), datetime.datetime.now().strftime("%m_%d_%Y_%I:%M:%S%p"))


    if ast.literal_eval(config.get("exp_details", "save")):
        if not os.path.exists(dir_name):
            os.makedirs(dir_name)

        # copy cfg file to experiment directory for record keeping purposes
        shutil.copy(cfg_fn, dir_name)

    return config, dir_name

    # print config.sections()
    # print config.get("NN", "type")
    # print json.loads(config.get("NN", "network_size"))
    # print config.get("exp_details", "name")


def test_train_split(X, Y, training_size=0.8):
    # X = X/360.
    trainXs, testXs  = [], []

    trainXs.append( X[:X.shape[0]*training_size,:].astype(np.float32))
    testXs.append( X[X.shape[0]*training_size:,:].astype(np.float32))

    # Could be a list??? Comiter thinks so....
    trainY = Y[:Y.shape[0]*training_size,:].astype(np.float32)
    testY = Y[Y.shape[0]*training_size:,:].astype(np.float32)

    return trainXs, trainY, testXs, testY


def parseParams(params):
    new_params = {}
    for param,_ in params.iteritems():
        if type(params[param]) == list:
            new_params[param] = [params[param].__str__()]
        elif type(params[param]) == dict:
            for k,v in params[param].iteritems():
                if type(v) == list:
                    new_v = v.__str__()
                else:
                    new_v = v
                new_params[k] = [new_v]
        else:
            new_params[param] = [params[param]]

    return new_params


def create_param_dict(config):
    params = {}
    params['NN__type'] = config.get("NN", "type")
    params['NN__network_size'] = json.loads(config.get("NN", "network_size"))
    params['NN__n_epochs'] = int(config.get("NN", "n_epochs"))
    params['NN__batchsize'] = int(config.get("NN", "batchsize"))
    params['NN__take_max'] = ast.literal_eval(config.get("NN", "take_max"))
    if config.has_option("data", "data_dist"):
        params['data__data_dist'] = config.get("data", "data_dist")
    else:
        params['data__data_dist'] = "random"
    if config.has_option("data", "epsilon"):
        params['NN__epsilon'] = float(config.get("NN", "epsilon"))
    else:
        params['NN__epsilon'] = 0.
    params['data__num_pts'] = int(config.get("data", "num_pts"))
    params['data__ndims'] = int(config.get("data", "ndims"))
    params['data__num_stations'] = int(config.get("data", "num_stations"))
    params['data__bs_type'] = config.get("data", "bs_type")
    params['exp_details__setname'] = config.get("exp_details", "setname")
    params['exp_details__name'] = config.get("exp_details", "name")<commit_msg>error fixing for epsilon<commit_after>import ConfigParser
import json
import datetime
import os
import shutil
import numpy as np
import ast

# TODO: convert output to dictionary
def load_configuration(cfg_fn):
    config = ConfigParser.ConfigParser()

    config.read(cfg_fn)

    if not os.path.exists("experiment_results"):
        os.makedirs("experiment_results")

    try:
        set_name = "%s" % (config.get("exp_details", "setname"))
    except:
        set_name = "anonymous_set"

    if not os.path.exists("experiment_results/" + set_name):
        os.makedirs("experiment_results/" + set_name)

    dir_name = "experiment_results/%s/%s__%s/" % (set_name, config.get("exp_details", "name"), datetime.datetime.now().strftime("%m_%d_%Y_%I:%M:%S%p"))


    if ast.literal_eval(config.get("exp_details", "save")):
        if not os.path.exists(dir_name):
            os.makedirs(dir_name)

        # copy cfg file to experiment directory for record keeping purposes
        shutil.copy(cfg_fn, dir_name)

    return config, dir_name

    # print config.sections()
    # print config.get("NN", "type")
    # print json.loads(config.get("NN", "network_size"))
    # print config.get("exp_details", "name")


def test_train_split(X, Y, training_size=0.8):
    # X = X/360.
    trainXs, testXs  = [], []

    trainXs.append( X[:X.shape[0]*training_size,:].astype(np.float32))
    testXs.append( X[X.shape[0]*training_size:,:].astype(np.float32))

    # Could be a list??? Comiter thinks so....
    trainY = Y[:Y.shape[0]*training_size,:].astype(np.float32)
    testY = Y[Y.shape[0]*training_size:,:].astype(np.float32)

    return trainXs, trainY, testXs, testY


def parseParams(params):
    new_params = {}
    for param,_ in params.iteritems():
        if type(params[param]) == list:
            new_params[param] = [params[param].__str__()]
        elif type(params[param]) == dict:
            for k,v in params[param].iteritems():
                if type(v) == list:
                    new_v = v.__str__()
                else:
                    new_v = v
                new_params[k] = [new_v]
        else:
            new_params[param] = [params[param]]

    return new_params


def create_param_dict(config):
    params = {}
    params['NN__type'] = config.get("NN", "type")
    params['NN__network_size'] = json.loads(config.get("NN", "network_size"))
    params['NN__n_epochs'] = int(config.get("NN", "n_epochs"))
    params['NN__batchsize'] = int(config.get("NN", "batchsize"))
    params['NN__take_max'] = ast.literal_eval(config.get("NN", "take_max"))
    if config.has_option("data", "data_dist"):
        params['data__data_dist'] = config.get("data", "data_dist")
    else:
        params['data__data_dist'] = "random"
    if config.has_option("NN", "epsilon"):
        params['NN__epsilon'] = float(config.get("NN", "epsilon"))
    else:
        params['NN__epsilon'] = 0.
    params['data__num_pts'] = int(config.get("data", "num_pts"))
    params['data__ndims'] = int(config.get("data", "ndims"))
    params['data__num_stations'] = int(config.get("data", "num_stations"))
    params['data__bs_type'] = config.get("data", "bs_type")
    params['exp_details__setname'] = config.get("exp_details", "setname")
    params['exp_details__name'] = config.get("exp_details", "name")
{'fraction_non_alphanumeric': 0.1649734647460197, 'fraction_numerical': 0.005458680818802123, 'mean_word_length': 3.3914780292942743, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1172203', 'n_tokens_mistral': 2269, 'n_tokens_neox': 2222, 'n_words': 400}