<commit_before>import unittest
from os.path import join

from ecnet.utils.data_utils import DataFrame
from ecnet.utils.server_utils import default_config
from ecnet.tasks.tuning import tune_hyperparameters


DB_LOC = 'cn_model_v1.0.csv'


class TestTune(unittest.TestCase):

    def test_tune_hyperparameters(self):

        print('\nUNIT TEST: tune_hyperparameters')
        df = DataFrame(DB_LOC)
        df.create_sets(random=True)
        config = default_config()
        new_hp = tune_hyperparameters(df, config, 2, 1)
        self.assertGreaterEqual(new_hp['beta_1'], 0)
        self.assertLessEqual(new_hp['beta_1'], 1)
        self.assertGreaterEqual(new_hp['beta_2'], 0)
        self.assertLessEqual(new_hp['beta_2'], 1)
        self.assertGreaterEqual(new_hp['decay'], 0)
        self.assertLessEqual(new_hp['decay'], 1)
        self.assertGreaterEqual(new_hp['epsilon'], 0)
        self.assertLessEqual(new_hp['epsilon'], 1)
        self.assertGreaterEqual(new_hp['learning_rate'], 0)
        self.assertLessEqual(new_hp['learning_rate'], 1)
        self.assertGreaterEqual(new_hp['batch_size'], 1)
        self.assertLessEqual(new_hp['batch_size'], len(df.learn_set))
        self.assertGreaterEqual(new_hp['hidden_layers'][0][0], 1)
        self.assertLessEqual(new_hp['hidden_layers'][0][0], 100)
        self.assertGreaterEqual(new_hp['hidden_layers'][1][0], 1)
        self.assertLessEqual(new_hp['hidden_layers'][1][0], 100)


if __name__ == '__main__':

    DB_LOC = join('../', 'cn_model_v1.0.csv')
    unittest.main()<commit_msg>Added multiprocessing unit test for tuning function<commit_after>import unittest
from os.path import join

from ecnet.utils.data_utils import DataFrame
from ecnet.utils.server_utils import default_config
from ecnet.tasks.tuning import tune_hyperparameters


DB_LOC = 'cn_model_v1.0.csv'


class TestTune(unittest.TestCase):

    def test_tune_hyperparameters(self):

        print('\nUNIT TEST: tune_hyperparameters')
        df = DataFrame(DB_LOC)
        df.create_sets(random=True)
        config = default_config()
        new_hp = tune_hyperparameters(df, config, 2, 1, epochs=100)
        self.assertGreaterEqual(new_hp['beta_1'], 0)
        self.assertLessEqual(new_hp['beta_1'], 1)
        self.assertGreaterEqual(new_hp['beta_2'], 0)
        self.assertLessEqual(new_hp['beta_2'], 1)
        self.assertGreaterEqual(new_hp['decay'], 0)
        self.assertLessEqual(new_hp['decay'], 1)
        self.assertGreaterEqual(new_hp['epsilon'], 0)
        self.assertLessEqual(new_hp['epsilon'], 1)
        self.assertGreaterEqual(new_hp['learning_rate'], 0)
        self.assertLessEqual(new_hp['learning_rate'], 1)
        self.assertGreaterEqual(new_hp['batch_size'], 1)
        self.assertLessEqual(new_hp['batch_size'], len(df.learn_set))
        self.assertGreaterEqual(new_hp['hidden_layers'][0][0], 1)
        self.assertLessEqual(new_hp['hidden_layers'][0][0], 600)
        self.assertGreaterEqual(new_hp['hidden_layers'][1][0], 1)
        self.assertLessEqual(new_hp['hidden_layers'][1][0], 600)

    def test_th_multiprocess(self):

        print('\nUNIT TEST: tune_hyperparameters multiprocessed')
        df = DataFrame(DB_LOC)
        df.create_sets(random=True)
        config = default_config()
        new_hp = tune_hyperparameters(df, config, 2, 1, 2, epochs=100)
        self.assertGreaterEqual(new_hp['beta_1'], 0)
        self.assertLessEqual(new_hp['beta_1'], 1)
        self.assertGreaterEqual(new_hp['beta_2'], 0)
        self.assertLessEqual(new_hp['beta_2'], 1)
        self.assertGreaterEqual(new_hp['decay'], 0)
        self.assertLessEqual(new_hp['decay'], 1)
        self.assertGreaterEqual(new_hp['epsilon'], 0)
        self.assertLessEqual(new_hp['epsilon'], 1)
        self.assertGreaterEqual(new_hp['learning_rate'], 0)
        self.assertLessEqual(new_hp['learning_rate'], 1)
        self.assertGreaterEqual(new_hp['batch_size'], 1)
        self.assertLessEqual(new_hp['batch_size'], len(df.learn_set))
        self.assertGreaterEqual(new_hp['hidden_layers'][0][0], 1)
        self.assertLessEqual(new_hp['hidden_layers'][0][0], 600)
        self.assertGreaterEqual(new_hp['hidden_layers'][1][0], 1)
        self.assertLessEqual(new_hp['hidden_layers'][1][0], 600)


if __name__ == '__main__':

    DB_LOC = join('../', 'cn_model_v1.0.csv')
    unittest.main()
{'fraction_non_alphanumeric': 0.13835174953959484, 'fraction_numerical': 0.026243093922651933, 'mean_word_length': 4.017321016166282, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1547502', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1464, 'n_words': 208}