<commit_before>from random import sample

from django.db.models import Q
from kolibri.content import models, serializers
from rest_framework import filters, pagination, viewsets


class ChannelMetadataCacheViewSet(viewsets.ModelViewSet):
    serializer_class = serializers.ChannelMetadataCacheSerializer

    def get_queryset(self):
        return models.ChannelMetadataCache.objects.all()


class ContentNodeFilter(filters.FilterSet):
    search = filters.django_filters.MethodFilter(action='title_description_filter')
    recommendations_for = filters.django_filters.MethodFilter()
    recommendations = filters.django_filters.MethodFilter()

    class Meta:
        model = models.ContentNode
        fields = ['parent', 'search', 'prerequisite_for', 'has_prerequisite', 'related', 'recommendations_for', 'recommendations']

    def title_description_filter(self, queryset, value):
        """
        search for title or description that contains the keywords that are not necessary in adjacent
        """
        exact_match = queryset.filter(Q(parent__isnull=False), Q(title__icontains=value) | Q(description__icontains=value))
        if exact_match:
            return exact_match
        # if no exact match, search for non-adjacent match
        return queryset.filter(
            Q(parent__isnull=False),
            reduce(lambda x, y: x & y, [Q(title__icontains=word) for word in value.split()]) |
            reduce(lambda x, y: x & y, [Q(description__icontains=word) for word in value.split()]))

    def filter_recommendations_for(self, queryset, value):
        recc_node = queryset.get(pk=value)
        descendants = recc_node.get_descendants(include_self=False).exclude(kind__in=['topic', ''])
        siblings = recc_node.get_siblings(include_self=False).exclude(kind__in=['topic', ''])
        data = descendants | siblings  # concatenates different querysets
        return data

    def filter_recommendations(self, queryset, value):<commit_msg>Fix non-direct match content search in Python 3.<commit_after>from functools import reduce
from random import sample

from django.db.models import Q
from kolibri.content import models, serializers
from rest_framework import filters, pagination, viewsets


class ChannelMetadataCacheViewSet(viewsets.ModelViewSet):
    serializer_class = serializers.ChannelMetadataCacheSerializer

    def get_queryset(self):
        return models.ChannelMetadataCache.objects.all()


class ContentNodeFilter(filters.FilterSet):
    search = filters.django_filters.MethodFilter(action='title_description_filter')
    recommendations_for = filters.django_filters.MethodFilter()
    recommendations = filters.django_filters.MethodFilter()

    class Meta:
        model = models.ContentNode
        fields = ['parent', 'search', 'prerequisite_for', 'has_prerequisite', 'related', 'recommendations_for', 'recommendations']

    def title_description_filter(self, queryset, value):
        """
        search for title or description that contains the keywords that are not necessary in adjacent
        """
        exact_match = queryset.filter(Q(parent__isnull=False), Q(title__icontains=value) | Q(description__icontains=value))
        if exact_match:
            return exact_match
        # if no exact match, search for non-adjacent match

        return queryset.filter(
            Q(parent__isnull=False),
            reduce(lambda x, y: x & y, [Q(title__icontains=word) for word in value.split()]) |
            reduce(lambda x, y: x & y, [Q(description__icontains=word) for word in value.split()]))

    def filter_recommendations_for(self, queryset, value):
        recc_node = queryset.get(pk=value)
        descendants = recc_node.get_descendants(include_self=False).exclude(kind__in=['topic', ''])
        siblings = recc_node.get_siblings(include_self=False).exclude(kind__in=['topic', ''])
        data = descendants | siblings  # concatenates different querysets
        return data

    def filter_recommendations(self, queryset, value):
{'fraction_non_alphanumeric': 0.09383578737209883, 'fraction_numerical': 0.0002495632642874969, 'mean_word_length': 4.05422446406053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6777387', 'n_tokens_mistral': 1129, 'n_tokens_neox': 1104, 'n_words': 264}