<commit_before>import itertools
import operator

import numpy
from pandas import DataFrame

from etl import ETLUtils
from tripadvisor.fourcity.user import User
from utils.constants import Constants

__author__ = 'fpena'


def remove_empty_user_reviews(reviews):
    """
    Returns a copy of the original reviews list without the reviews made by
    users who have an empty ID

    :param reviews: a list of reviews
    :return: a copy of the original reviews list without the reviews made by
    users who have an empty ID
    """
    filtered_reviews = [review for review in reviews if
                        review[Constants.USER_ID_FIELD] != '' and
                        review[Constants.USER_ID_FIELD] != 'null']
    return filtered_reviews


def extract_fields(reviews):
    """
    Modifies the given list of reviews in order to extract the values contained
    in the ratings field to top level fields. For instance, a review which is
    in the form
    {'user_id': 'U1', 'offering_id': :'I1',
    'ratings': {'cleanliness': 4.0, 'location': 5.0}}
    would become:

    {'user_id': 'U1', 'offering_id': :'I1',
    'ratings': {'cleanliness': 4.0, 'location': 5.0},
    'cleanliness_rating': 4.0, 'location_rating': 5.0}

    :param reviews: a list of reviews.
    """

    ratings_criteria = [
        'cleanliness',
        'location',
        'rooms',
        'service',
        # 'sleep_quality',
        'value'
    ]

    for review in reviews:
        review[Constants.USER_ID_FIELD] = review['author']['id']
        review['overall_rating'] = review['ratings']['overall']

        ratings = review['ratings']
        multi_ratings = []
        for criterion in ratings_criteria:
            if criterion in ratings:
                # review[criterion + '_rating'] = review['ratings'][criterion]
                multi_ratings.append(review['ratings'][criterion])
            else:
                multi_ratings.append(-1)

        review['multi_ratings'] = multi_ratings


def remove_users_with_low_reviews(reviews, min_reviews):
    """
    Returns a copy of the original reviews list without the reviews made by
    users who have made less than min_reviews reviews

    :param reviews: a list of reviews
    :param min_reviews: the minimum number of reviews a user must have in order
    not to be removed from the reviews list
    :return: a copy of the original reviews list without the reviews made by
    users who have made less than min_reviews reviews
    """
    users = get_user_list(reviews, min_reviews)
    return ETLUtils.filter_records(reviews, Constants.USER_ID_FIELD, users)


def remove_items_with_low_reviews(reviews, min_reviews):
    """
    Returns a copy of the original reviews list without the reviews of hotels
    that just have been reviewed once

    :param reviews: a list of reviews
    :return: a copy of the original reviews list without the reviews of hotels
    that just have been reviewed once
    """
    items = get_item_list(reviews, min_reviews)
    return ETLUtils.filter_records(reviews, Constants.ITEM_ID_FIELD, items)


def remove_missing_ratings_reviews(reviews):
    """
    Returns a copy of the original reviews list without the reviews that have
    missing ratings

    :param reviews: a list of reviews
    :return: a copy of the original reviews list without the reviews that have
    missing ratings
    """
    filtered_reviews = [review for review in reviews if
                        verify_rating_criteria(review)]
    return filtered_reviews<commit_msg>Added a bit of logging information to the remove_items_with_low_reviews() function of the extractor module<commit_after>import itertools
import operator

import numpy
from pandas import DataFrame

from etl import ETLUtils
from tripadvisor.fourcity.user import User
from utils.constants import Constants

__author__ = 'fpena'


def remove_empty_user_reviews(reviews):
    """
    Returns a copy of the original reviews list without the reviews made by
    users who have an empty ID

    :param reviews: a list of reviews
    :return: a copy of the original reviews list without the reviews made by
    users who have an empty ID
    """
    filtered_reviews = [review for review in reviews if
                        review[Constants.USER_ID_FIELD] != '' and
                        review[Constants.USER_ID_FIELD] != 'null']
    return filtered_reviews


def extract_fields(reviews):
    """
    Modifies the given list of reviews in order to extract the values contained
    in the ratings field to top level fields. For instance, a review which is
    in the form
    {'user_id': 'U1', 'offering_id': :'I1',
    'ratings': {'cleanliness': 4.0, 'location': 5.0}}
    would become:

    {'user_id': 'U1', 'offering_id': :'I1',
    'ratings': {'cleanliness': 4.0, 'location': 5.0},
    'cleanliness_rating': 4.0, 'location_rating': 5.0}

    :param reviews: a list of reviews.
    """

    ratings_criteria = [
        'cleanliness',
        'location',
        'rooms',
        'service',
        # 'sleep_quality',
        'value'
    ]

    for review in reviews:
        review[Constants.USER_ID_FIELD] = review['author']['id']
        review['overall_rating'] = review['ratings']['overall']

        ratings = review['ratings']
        multi_ratings = []
        for criterion in ratings_criteria:
            if criterion in ratings:
                # review[criterion + '_rating'] = review['ratings'][criterion]
                multi_ratings.append(review['ratings'][criterion])
            else:
                multi_ratings.append(-1)

        review['multi_ratings'] = multi_ratings


def remove_users_with_low_reviews(reviews, min_reviews):
    """
    Returns a copy of the original reviews list without the reviews made by
    users who have made less than min_reviews reviews

    :param reviews: a list of reviews
    :param min_reviews: the minimum number of reviews a user must have in order
    not to be removed from the reviews list
    :return: a copy of the original reviews list without the reviews made by
    users who have made less than min_reviews reviews
    """
    users = get_user_list(reviews, min_reviews)
    return ETLUtils.filter_records(reviews, Constants.USER_ID_FIELD, users)


def remove_items_with_low_reviews(reviews, min_reviews):
    """
    Returns a copy of the original reviews list without the reviews of hotels
    that just have been reviewed once

    :param reviews: a list of reviews
    :return: a copy of the original reviews list without the reviews of hotels
    that just have been reviewed once
    """
    items = get_item_list(reviews, min_reviews)
    all_items = get_groupby_list(reviews, Constants.ITEM_ID_FIELD)
    num_discarded_items = len(all_items) - len(items)
    print('Discarded %d items due to low count' % num_discarded_items)
    return ETLUtils.filter_records(reviews, Constants.ITEM_ID_FIELD, items)


def remove_missing_ratings_reviews(reviews):
    """
    Returns a copy of the original reviews list without the reviews that have
    missing ratings

    :param reviews: a list of reviews
    :return: a copy of the original reviews list without the reviews that have
    missing ratings
    """
    filtered_reviews = [review for review in reviews if
                        verify_rating_criteria(review)]
    return filtered_reviews
{'fraction_non_alphanumeric': 0.07849272207862876, 'fraction_numerical': 0.004625221058359407, 'mean_word_length': 3.044004400440044, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6639148', 'n_tokens_mistral': 2095, 'n_tokens_neox': 2033, 'n_words': 746}