<commit_before>from users.models import User
from catalog.models import Course
import collections
from django.contrib.contenttypes.models import ContentType
from actstream.models import Follow


def distance(v1, v2):
    absolute_difference = [abs(c1 - c2) for c1, c2 in zip(v1, v2)]
    distance = sum(absolute_difference)
    return distance


def get_users_following_dict():
    course_type = ContentType.objects.get(app_label="catalog", model="course")
    follows = Follow.objects\
        .filter(content_type=course_type)\
        .select_related('user')\
        .prefetch_related('follow_object')

    following_dict = collections.defaultdict(set)
    for follow in follows:
        following_dict[follow.user.netid].add(follow.follow_object)

    return following_dict


def suggest(target_user, K=15):
    courses = Course.objects.all()
    users = {user.netid: user for user in User.objects.all()}
    users_following = get_users_following_dict()

    vectors = {}
    for netid, user in users.items():
        following = users_following[netid]
        vectors[netid] = [course in following for course in courses]

    target_vector = vectors[target_user.netid]

    distances = {netid: distance(target_vector, vector) for netid, vector in vectors.items()}
    non_null_distances = {netid: distance for netid, distance in distances.items() if distance > 0}

    get_score = lambda x: x[1]
    neighbors = sorted(non_null_distances.items(), key=get_score)[:K]

    best_matches = collections.Counter()
    target_set = users_following[target_user.netid]

    for netid, score in neighbors:
        differences = users_following[netid] - target_set
        best_matches.update(differences)

    return best_matches<commit_msg>Use only user_id and course_id during the kNN computation: speedup is 5x Previously most time was consumed in db queries and suprisingly in __hash__ methods (seem that hashing a django model takes longer that hashing a int)<commit_after>from catalog.models import Course
import collections
from django.contrib.contenttypes.models import ContentType
from actstream.models import Follow


def distance(v1, v2):
    absolute_difference = [abs(c1 - c2) for c1, c2 in zip(v1, v2)]
    distance = sum(absolute_difference)
    return distance


def get_users_following_dict():
    course_type = ContentType.objects.get(app_label="catalog", model="course")
    follows = Follow.objects.filter(content_type=course_type).only('user_id', 'object_id')

    following_dict = collections.defaultdict(set)
    for follow in follows:
        following_dict[follow.user_id].add(int(follow.object_id))

    return following_dict


def suggest(target_user, K=15):
    courses = Course.objects.only('id')
    users_following = get_users_following_dict()

    vectors = {}
    for user_id, following in users_following.items():
        vectors[user_id] = [course.id in following for course in courses]

    target_vector = vectors[target_user.id]

    distances = {user_id: distance(target_vector, vector) for user_id, vector in vectors.items()}
    non_null_distances = {user_id: distance for user_id, distance in distances.items() if distance > 0}

    get_score = lambda x: x[1]
    neighbors = sorted(non_null_distances.items(), key=get_score)[:K]

    best_matches = collections.Counter()
    target_set = users_following[target_user.id]

    for user_id, score in neighbors:
        differences = users_following[user_id] - target_set
        best_matches.update(differences)

    return [(Course.objects.get(id=course_id), hits) for course_id, hits in best_matches.most_common()]
{'fraction_non_alphanumeric': 0.09355913381454747, 'fraction_numerical': 0.00694058856191005, 'mean_word_length': 4.244541484716157, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7210947', 'n_tokens_mistral': 1097, 'n_tokens_neox': 1058, 'n_words': 280}