<commit_before>import time

from hyperopt import fmin
from hyperopt import hp
from hyperopt import tpe
from hyperopt.mongoexp import MongoTrials


def fibonacci(n):

    a = 1
    b = 1

    sequence = [a, b]

    if n < 2:
        return 1

    for i in range(2, n):
        b_temp = b
        b += a
        a = b_temp
        sequence.append(b)

    return sequence


def run_recommender(args):
    import sys
    # sys.path.append('/Users/fpena/UCC/Thesis/projects/yelp/source/python')
    sys.path.append('/home/fpena/yelp/source/python')
    from utils.constants import Constants
    from evaluation.context_top_n_runner import ContextTopNRunner

    print('\n\n************************\n************************\n')
    print('args', args)

    # Cast integer values
    # args['fm_iterations'] = int(args['fm_iterations'])
    args['fm_num_factors'] = int(args['fm_num_factors'])
    if args['use_context']:
        args['lda_model_iterations'] = int(args['lda_model_iterations'])
        args['lda_model_passes'] = int(args['lda_model_passes'])
        args['lda_num_topics'] = int(args['lda_num_topics'])

    Constants.update_properties(args)

    # Finish updating parameters

    my_context_top_n_runner = ContextTopNRunner()
    results = my_context_top_n_runner.run()
    results['loss'] = -results[Constants.EVALUATION_METRIC]
    results['status'] = 'ok'

    print('loss', results['loss'])

    return results


def tune_parameters():

    # trials = Trials()
    from utils.constants import Constants

    context_name = '_context' if Constants.USE_CONTEXT else '_nocontext'
    cycle = '_' + str(Constants.NESTED_CROSS_VALIDATION_CYCLE)

    mongo_url =\
        'mongo://localhost:1234/' +\
        Constants.ITEM_TYPE + context_name + '_db_nested' + cycle + '/jobs'
    trials = MongoTrials(mongo_url, exp_key='exp1')

    print('Connected to %s' % mongo_url)

    params = Constants.get_properties_copy()
    params.update({
        'business_type': Constants.ITEM_TYPE,
        'topn_num_items': Constants.TOPN_NUM_ITEMS,
        'nested_cross_validation_cycle': Constants.NESTED_CROSS_VALIDATION_CYCLE,
        # 'fm_init_stdev': hp.uniform('fm_init_stdev', 0, 2),
        # 'fm_iterations': hp.quniform('fm_context_iterations', 100, 500, 1),
        'fm_num_factors': hp.quniform('fm_context_num_factors', 0, 200, 1),
        # 'fm_use_1way_interactions': hp.choice('fm_use_1way_interactions', [True, False]),
        # 'fm_use_bias': hp.choice('use_bias', [True, False]),
        # 'lda_alpha': hp.uniform('lda_alpha', 0, 1),
        # 'lda_beta': hp.uniform('lda_beta', 0, 2),
        # 'lda_epsilon': hp.uniform('lda_epsilon', 0, 0.5),
        'lda_model_iterations': hp.quniform('lda_model_iterations', 50, 500, 1),
        'lda_model_passes': hp.quniform('lda_model_passes', 1, 100, 1),
        'lda_num_topics': hp.quniform('lda_num_topics', 1, 1000, 1),
        # 'topic_weighting_method': hp.choice('topic_weighting_method', ['probability', 'binary', 'all_topics']),
        # 'use_no_context_topics_sum': hp.choice('use_no_context_topics_sum', [True, False]),
        'use_context': Constants.USE_CONTEXT
    })

    space =\
        hp.choice('use_context', [
            params,
        ])

    if not Constants.USE_CONTEXT:
        unwanted_args = [
            'lda_epsilon',
            'lda_model_iterations',
            'lda_model_passes',
            'lda_num_topics'
        ]

        for element in space.pos_args[1].named_args[:]:
            if element[0] in unwanted_args:
                space.pos_args[1].named_args.remove(element)

    # best = fmin(
    #     run_recommender, space=space, algo=tpe.suggest,
    #     max_evals=100, trials=trials)

    print('losses', sorted(trials.losses()))
    print(
        'best', trials.best_trial['result']['loss'],
        trials.best_trial['misc']['vals'])
    print('num trials: %d' % len(trials.losses()))


start = time.time()
tune_parameters()
end = time.time()
total_time = end - start
print("Total time = %f seconds" % total_time)
<commit_msg>Updated the values for the hyperparameters that have to be tuned. This time they are designed to work with the restaurants dataset<commit_after>import time

from hyperopt import fmin
from hyperopt import hp
from hyperopt import tpe
from hyperopt.mongoexp import MongoTrials


def fibonacci(n):

    a = 1
    b = 1

    sequence = [a, b]

    if n < 2:
        return 1

    for i in range(2, n):
        b_temp = b
        b += a
        a = b_temp
        sequence.append(b)

    return sequence


def run_recommender(args):
    import sys
    # sys.path.append('/Users/fpena/UCC/Thesis/projects/yelp/source/python')
    sys.path.append('/home/fpena/yelp/source/python')
    from utils.constants import Constants
    from evaluation.context_top_n_runner import ContextTopNRunner

    print('\n\n************************\n************************\n')
    print('args', args)

    # Cast integer values
    # args['fm_iterations'] = int(args['fm_iterations'])
    args['fm_num_factors'] = int(args['fm_num_factors'])
    if args['use_context']:
        args['lda_model_iterations'] = int(args['lda_model_iterations'])
        args['lda_model_passes'] = int(args['lda_model_passes'])
        args['lda_num_topics'] = int(args['lda_num_topics'])

    Constants.update_properties(args)

    # Finish updating parameters

    my_context_top_n_runner = ContextTopNRunner()
    results = my_context_top_n_runner.run()
    results['loss'] = -results[Constants.EVALUATION_METRIC]
    results['status'] = 'ok'

    print('loss', results['loss'])

    return results


def tune_parameters():

    # trials = Trials()
    from utils.constants import Constants

    context_name = '_context' if Constants.USE_CONTEXT else '_nocontext'
    cycle = '_' + str(Constants.NESTED_CROSS_VALIDATION_CYCLE)

    mongo_url =\
        'mongo://localhost:1234/' +\
        Constants.ITEM_TYPE + context_name + '_db_nested' + cycle + '/jobs'
    trials = MongoTrials(mongo_url, exp_key='exp1')

    print('Connected to %s' % mongo_url)

    params = Constants.get_properties_copy()
    params.update({
        'business_type': Constants.ITEM_TYPE,
        'topn_num_items': Constants.TOPN_NUM_ITEMS,
        'nested_cross_validation_cycle': Constants.NESTED_CROSS_VALIDATION_CYCLE,
        # 'fm_init_stdev': hp.uniform('fm_init_stdev', 0, 2),
        # 'fm_iterations': hp.quniform('fm_context_iterations', 100, 500, 1),
        'fm_num_factors': hp.quniform('fm_context_num_factors', 0, 200, 1),
        # 'fm_use_1way_interactions': hp.choice('fm_use_1way_interactions', [True, False]),
        # 'fm_use_bias': hp.choice('use_bias', [True, False]),
        # 'lda_alpha': hp.uniform('lda_alpha', 0, 1),
        # 'lda_beta': hp.uniform('lda_beta', 0, 2),
        # 'lda_epsilon': hp.uniform('lda_epsilon', 0, 0.5),
        'lda_model_iterations': hp.quniform('lda_model_iterations', 50, 500, 1),
        'lda_model_passes': hp.quniform('lda_model_passes', 1, 100, 1),
        # 'lda_num_topics': hp.quniform('lda_num_topics', 1, 1000, 1),
        'lda_num_topics': hp.choice('lda_num_topics', [10, 20, 30, 50, 75, 100, 150, 300]),
        'topic_model_type': hp.choice('topic_model_type', ['lda', 'mnf']),
        # 'topic_weighting_method': hp.choice('topic_weighting_method', ['probability', 'binary', 'all_topics']),
        # 'use_no_context_topics_sum': hp.choice('use_no_context_topics_sum', [True, False]),
        'use_context': Constants.USE_CONTEXT
    })

    space =\
        hp.choice('use_context', [
            params,
        ])

    if not Constants.USE_CONTEXT:
        unwanted_args = [
            'lda_epsilon',
            'lda_model_iterations',
            'lda_model_passes',
            'lda_num_topics'
        ]

        for element in space.pos_args[1].named_args[:]:
            if element[0] in unwanted_args:
                space.pos_args[1].named_args.remove(element)

    # best = fmin(
    #     run_recommender, space=space, algo=tpe.suggest,
    #     max_evals=100, trials=trials)

    print('losses', sorted(trials.losses()))
    print(
        'best', trials.best_trial['result']['loss'],
        trials.best_trial['misc']['vals'])
    print('num trials: %d' % len(trials.losses()))


start = time.time()
tune_parameters()
end = time.time()
total_time = end - start
print("Total time = %f seconds" % total_time)

{'fraction_non_alphanumeric': 0.14701670644391407, 'fraction_numerical': 0.015632458233890215, 'mean_word_length': 3.280388151174668, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5389209', 'n_tokens_mistral': 3066, 'n_tokens_neox': 2868, 'n_words': 521}