<commit_before>from collections import defaultdict
from datetime import datetime
from datetime import timedelta
import csv
import mongoengine as me
import rmc.models as m
import rmc.shared.constants as c
import rmc.shared.util as _util
import sys
import time

def truncate_datetime(dt):
    return dt - timedelta(
            hours=dt.hour,
            minutes=dt.minute,
            seconds=dt.second,
            microseconds=dt.microsecond)

def generic_stats(show_all=False):
    users = m.User.objects()
    ucs = m.UserCourse.objects()

    num_users = len(users)

    num_users_with_transcript = sum(
            [1 if user.transcripts_imported else 0 for user in users])

    num_ucs = len(ucs)

    if show_all:
        # UserCourse Course/Prof Reviews/Ratings
        uc_crev = 0
        uc_prev = 0
        uc_crat = 0
        uc_prat = 0
        for uc in ucs:
            cr = uc.course_review
            pr = uc.professor_review
            if cr.comment:
                uc_crev += 1
            if cr.interest:
                uc_crat += 1
            if cr.easiness:
                uc_crat += 1
            if cr.usefulness:
                uc_crat += 1
            if pr.comment:
                uc_prev += 1
            if pr.clarity:
                uc_prat += 1
            if pr.passion:
                uc_prat += 1

    today = datetime.now() - timedelta(hours=24)
    signups = users_joined_after()

    result = {
        'num_users': len(users),
        'num_users_with_transcript': num_users_with_transcript,
        'num_ucs': len(ucs),
        'num_signups_today': signups,
        'num_signups_start_time': today,
        'epoch': datetime.now(),
    }
    if show_all:
        result.update({
            'num_reviews': uc_crev + uc_prev,
            'num_ratings': uc_crat + uc_prat,
            'num_course_reviews': uc_crev,
            'num_professor_reviews': uc_prev,
            'num_course_ratings': uc_crat,
            'num_professor_ratings': uc_prat,
        })

    return result

def print_generic_stats():
    data = generic_stats(show_all=True)

    print """
Total User
%s
User with transcripts imported
%s
Total UserCourse
%s
Total UserCourse Course review
%s
Total UserCourse Prof reviews
%s
Total UserCourse Course ratings
%s
Total UserCourse Prof ratings
%s
Users signed up since a day ago (%s)
%s""" % (
        data['num_users'],
        data['num_users_with_transcript'],
        data['num_ucs'],
        data['num_course_reviews'],
        data['num_professor_reviews'],
        data['num_course_ratings'],
        data['num_professor_ratings'],
        data['num_signups_start_time'],
        data['num_signups_today'],
    )

def count_user_joined_date(cmp_op):
    """Count the number of users that cmp_op(join_date) returns true for"""
    users = m.User.objects()
    join_count = 0
    for user in users:
        if cmp_op(user.join_date):
            join_count += 1
    return join_count

def users_joined_after(date=None):
    if date is None:
        date = datetime.now() - timedelta(hours=24)
    return count_user_joined_date(date.__le__)

def users_joined_before(date=None):
    if date is None:
        date = datetime.now() - timedelta(hours=24)
    return count_user_joined_date(date.__ge__)

def latest_reviews(n=5):
    tups = []

    for uc in m.UserCourse.objects():
        cr_date = uc.course_review.comment_date
        pr_date = uc.professor_review.comment_date
        if cr_date:
            tups.append((cr_date, 'course_review', uc))
        if pr_date:
            tups.append((pr_date, 'professor_review', uc))

    tups.sort(reverse=True)

    result = []
    for tup in tups:
        date, rev_type, uc = tup
        result.append({
            'user_id': uc.user_id,
            'course_id': uc.course_id,
            'professor_id': uc.professor_id,
            'text': getattr(uc, rev_type).comment,
            'time': date,
            'type': rev_type,
        })
        if len(result) == n:
            break

    return result

def reviews_given(user):
    ucs = m.UserCourse.objects(user_id=user.id)
    review_count = 0
    for uc in ucs:
        if uc.course_review.comment:
            review_count += 1
        if uc.professor_review.comment:
            review_count += 1
    return review_count

def ratings_given(user):
    ucs = m.UserCourse.objects(user_id=user.id)
    rating_count = 0
    for uc in ucs:
	cr = uc.course_review
        if cr.interest:
            rating_count += 1
        if cr.easiness:
            rating_count += 1
        if cr.usefulness:<commit_msg>/dashboard loads in < 1 second; # ratings bug fix<commit_after># !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# There's some code in this file that checks for whether a field has been rated
# incorrectly (eg. `if course_review.interest`). Also, there's very slow code
# that loads all records in a collection into memory that should be converted
# to Mongo queries.
# TODO(david): Clean up this file and fix all the issues. Until then, please be
#     wary of what you use.
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!


from collections import defaultdict
from datetime import datetime, timedelta
import csv
import mongoengine as me
import rmc.models as m
import rmc.shared.constants as c
import rmc.shared.util as _util
import sys

from flask_debugtoolbar_lineprofilerpanel.profile import line_profile

def truncate_datetime(dt):
    return dt - timedelta(
            hours=dt.hour,
            minutes=dt.minute,
            seconds=dt.second,
            microseconds=dt.microsecond)

@line_profile
def generic_stats(show_all=False):
    num_ucs = m.UserCourse.objects().count()

    num_users = m.User.objects.count()
    num_users_with_transcript = m.User.objects(
            transcripts_imported__gt=0).count()

    num_course_reviews = m.UserCourse.objects(
            course_review__comment__ne='').count()
    num_professor_reviews = m.UserCourse.objects(
            professor_review__comment__ne='').count()

    # TODO(david): Make rating_fields a class method
    num_course_ratings = 0
    for rating in m.CourseReview().rating_fields():
        query = {}
        query['course_review__%s__ne' % rating] = None
        num_course_ratings += m.UserCourse.objects(**query).count()

    num_professor_ratings = 0
    for rating in m.ProfessorReview().rating_fields():
        query = {}
        query['professor_review__%s__ne' % rating] = None
        num_professor_ratings += m.UserCourse.objects(**query).count()

    yesterday = datetime.now() - timedelta(hours=24)
    signups = users_joined_after(yesterday)

    result = {
        'num_users': num_users,
        'num_users_with_transcript': num_users_with_transcript,
        'num_ucs': num_ucs,
        'num_signups_today': signups,
        'num_signups_start_time': yesterday,
        'epoch': datetime.now(),
    }
    if show_all:
        result.update({
            'num_reviews': num_course_reviews + num_professor_reviews,
            'num_ratings': num_course_ratings + num_professor_ratings,
            'num_course_reviews': num_course_reviews,
            'num_professor_reviews': num_professor_reviews,
            'num_course_ratings': num_course_ratings,
            'num_professor_ratings': num_professor_ratings,
        })

    return result

def print_generic_stats():
    data = generic_stats(show_all=True)

    print """
Total User
%s
User with transcripts imported
%s
Total UserCourse
%s
Total UserCourse Course review
%s
Total UserCourse Prof reviews
%s
Total UserCourse Course ratings
%s
Total UserCourse Prof ratings
%s
Users signed up since a day ago (%s)
%s""" % (
        data['num_users'],
        data['num_users_with_transcript'],
        data['num_ucs'],
        data['num_course_reviews'],
        data['num_professor_reviews'],
        data['num_course_ratings'],
        data['num_professor_ratings'],
        data['num_signups_start_time'],
        data['num_signups_today'],
    )

# Inclusive
def users_joined_after(date=None):
    if date is None:
        date = datetime.now() - timedelta(hours=24)
    return m.User.objects(join_date__gte=date).count()

# Exclusive
def users_joined_before(date=None):
    if date is None:
        date = datetime.now() - timedelta(hours=24)
    return m.User.objects(join_date__lt=date).count()

def latest_reviews(n=5):
    tups = []

    recent_ucs_by_course = m.UserCourse.objects().order_by(
            '-course_review__comment_date').limit(n)
    recent_ucs_by_prof = m.UserCourse.objects().order_by(
            '-prof_review__comment_date').limit(n)

    for uc in (set(recent_ucs_by_course) & set(recent_ucs_by_prof)):
        cr_date = uc.course_review.comment_date
        pr_date = uc.professor_review.comment_date
        if cr_date:
            tups.append((cr_date, 'course_review', uc))
        if pr_date:
            tups.append((pr_date, 'professor_review', uc))

    tups.sort(reverse=True)

    result = []
    for tup in tups:
        date, rev_type, uc = tup
        result.append({
            'user_id': uc.user_id,
            'course_id': uc.course_id,
            'professor_id': uc.professor_id,
            'text': getattr(uc, rev_type).comment,
            'date': date,
            'type': rev_type,
        })
        if len(result) == n:
            break

    return result

def reviews_given(user):
    ucs = m.UserCourse.objects(user_id=user.id)
    review_count = 0
    for uc in ucs:
        if uc.course_review.comment:
            review_count += 1
        if uc.professor_review.comment:
            review_count += 1
    return review_count

def ratings_given(user):
    ucs = m.UserCourse.objects(user_id=user.id)
    rating_count = 0
    for uc in ucs:
	cr = uc.course_review
        if cr.interest:
            rating_count += 1
        if cr.easiness:
            rating_count += 1
        if cr.usefulness:
{'fraction_non_alphanumeric': 0.11159100040176778, 'fraction_numerical': 0.004519887505022097, 'mean_word_length': 2.5121693121693123, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5241988', 'n_tokens_mistral': 3345, 'n_tokens_neox': 3109, 'n_words': 714}