<commit_before>import sys
from etl import ETLUtils
from evaluation import precision_in_top_n
from perfomancetest.context_recommender_tests import RMSE_HEADERS
from perfomancetest.context_recommender_tests import TOPN_HEADERS

sys.path.append('/Users/fpena/UCC/Thesis/projects/yelp/source/python')
import time
from functools import partial
from multiprocessing import Pool
import itertools
import numpy
import cPickle as pickle
from perfomancetest import context_recommender_tests
from tripadvisor.fourcity import recommender_evaluator

__author__ = 'fpena'


# This wrapper is necessary when you want to pass a function more
# than 1 parameter
def run_rmse_test_wrapper(args):
    return recommender_evaluator.perform_cross_validation(*args)

# This wrapper is necessary when you want to pass a function more
# than 1 parameter
def run_topn_test_wrapper(args):
    return precision_in_top_n.calculate_recall_in_top_n(*args)


def parallel_run_rmse_test(
        records_file, recommenders, binary_reviews_file, reviews_type=None):

    records = context_recommender_tests.load_records(records_file)
    # records = extractor.remove_users_with_low_reviews(records, 2)
    with open(binary_reviews_file, 'rb') as read_file:
        binary_reviews = pickle.load(read_file)

    if len(records) != len(binary_reviews):
        raise ValueError("The records and reviews should have the same length")

    num_folds = 5

    args = itertools.product(
        [records],
        recommenders,
        [num_folds],
        [binary_reviews],
        [reviews_type]
    )

    pool = Pool(processes=2)
    results_list = pool.map(run_rmse_test_wrapper, args)
    pool.close()
    pool.join()

    # After we have finished executing, we process the results
    dataset_info_map = {}
    dataset_info_map['dataset'] = records_file.split('/')[-1]<commit_msg>Added exception support so that functions executed in parallel also show exceptions<commit_after>import sys
import traceback
from etl import ETLUtils
from evaluation import precision_in_top_n
from perfomancetest.context_recommender_tests import RMSE_HEADERS
from perfomancetest.context_recommender_tests import TOPN_HEADERS

sys.path.append('/Users/fpena/UCC/Thesis/projects/yelp/source/python')
import time
from functools import partial
from multiprocessing import Pool
import itertools
import numpy
import cPickle as pickle
from perfomancetest import context_recommender_tests
from tripadvisor.fourcity import recommender_evaluator

__author__ = 'fpena'


# This wrapper is necessary when you want to pass a function more
# than 1 parameter
def run_rmse_test_wrapper(args):
    try:
        return recommender_evaluator.perform_cross_validation(*args)
    except Exception as e:
        print('Caught exception in worker thread')

        # This prints the type, value, and stack trace of the
        # current exception being handled.
        traceback.print_exc()

        print()
        raise e


# This wrapper is necessary when you want to pass a function more
# than 1 parameter
def run_topn_test_wrapper(args):
    try:
        return precision_in_top_n.calculate_recall_in_top_n(*args)
    except Exception as e:
        print('Caught exception in worker thread')

        # This prints the type, value, and stack trace of the
        # current exception being handled.
        traceback.print_exc()

        print()
        raise e


def parallel_run_rmse_test(
        records_file, recommenders, binary_reviews_file, reviews_type=None):

    records = context_recommender_tests.load_records(records_file)
    # records = extractor.remove_users_with_low_reviews(records, 2)
    with open(binary_reviews_file, 'rb') as read_file:
        binary_reviews = pickle.load(read_file)

    if len(records) != len(binary_reviews):
        raise ValueError("The records and reviews should have the same length")

    num_folds = 5

    args = itertools.product(
        [records],
        recommenders,
        [num_folds],
        [binary_reviews],
        [reviews_type]
    )

    pool = Pool(processes=2)
    results_list = pool.map(run_rmse_test_wrapper, args)
    pool.close()
    pool.join()

    # After we have finished executing, we process the results
    dataset_info_map = {}
    dataset_info_map['dataset'] = records_file.split('/')[-1]
{'fraction_non_alphanumeric': 0.06793857608189856, 'fraction_numerical': 0.002791996277338297, 'mean_word_length': 4.0046565774155995, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3109517', 'n_tokens_mistral': 1308, 'n_tokens_neox': 1280, 'n_words': 356}