<commit_before>import re
import haystack
from distutils.util import strtobool
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import FieldError
from haystack.backends.whoosh_backend import WhooshSearchBackend
from haystack.constants import DJANGO_CT, ITERATOR_LOAD_PER_QUERY
from haystack.query import SearchQuerySet
from haystack.utils import get_model_ct
from rest_framework import filters
from whoosh.qparser import QueryParser
from whoosh.query import Term, And

from .models import Asset
from .models.object_permission import get_objects_for_user, get_anonymous_user


class AssetOwnerFilterBackend(filters.BaseFilterBackend):
    """
    For use with AssetVersions
    Restricts access to items that are owned by the current user
    """
    def filter_queryset(self, request, queryset, view):
        return queryset.filter(asset__owner=request.user)


class KpiObjectPermissionsFilter(object):
    perm_format = '%(app_label)s.view_%(model_name)s'

    def filter_queryset(self, request, queryset, view):
        user = request.user
        if user.is_superuser and view.action != 'list':
            # For a list, we won't deluge the superuser with everyone else's
            # stuff. This isn't a list, though, so return it all
            return queryset
        # Governs whether unsubscribed (but publicly discoverable) objects are
        # included. Exclude them by default
        all_public = bool(strtobool(
            request.query_params.get('all_public', 'false').lower()))

        model_cls = queryset.model
        kwargs = {
            'app_label': model_cls._meta.app_label,
            'model_name': model_cls._meta.model_name,
        }
        permission = self.perm_format % kwargs

        if user.is_anonymous():
            user = get_anonymous_user()
            # Avoid giving anonymous users special treatment when viewing
            # public objects
            owned_and_explicitly_shared = queryset.none()
        else:
            owned_and_explicitly_shared = get_objects_for_user(
                user, permission, queryset)
        public = get_objects_for_user(
            get_anonymous_user(), permission, queryset)
        if view.action != 'list':
            # Not a list, so discoverability doesn't matter
            return owned_and_explicitly_shared | public

        # For a list, do not include public objects unless they are also
        # discoverable
        try:
            discoverable = public.filter(discoverable_when_public=True)
        except FieldError:
            try:
                # The model does not have a discoverability setting, but maybe
                # its parent does
                discoverable = public.filter(
                    parent__discoverable_when_public=True)
            except FieldError:
                # Neither the model or its parent has a discoverability setting
                discoverable = public.none()

        if all_public:
            # We were asked not to consider subscriptions; return all
            # discoverable objects
            return owned_and_explicitly_shared | discoverable

        # Of the discoverable objects, determine to which the user has
        # subscribed
        try:
            subscribed = public.filter(usercollectionsubscription__user=user)
        except FieldError:
            try:
                # The model does not have a subscription relation, but maybe
                # its parent does
                subscribed = public.filter(
                    parent__usercollectionsubscription__user=user)
            except FieldError:
                # Neither the model or its parent has a subscription relation
                subscribed = public.none()

        return owned_and_explicitly_shared | subscribed


class RelatedAssetPermissionsFilter(KpiObjectPermissionsFilter):
    ''' Uses KpiObjectPermissionsFilter to determine which assets the user
    may access, and then filters the provided queryset to include only objects
    related to those assets. The queryset's model must be related to `Asset`
    via a field named `asset`. '''

    def filter_queryset(self, request, queryset, view):
        available_assets = super(
            RelatedAssetPermissionsFilter, self
        ).filter_queryset(
            request=request,
            queryset=Asset.objects.all(),
            view=view
        )
        return queryset.filter(asset__in=available_assets)


class SearchFilter(filters.BaseFilterBackend):
    ''' Filter objects by searching with Whoosh if the request includes a `q`
    parameter. Another parameter, `parent`, is recognized when its value is an
    empty string; this restricts the queryset to objects without parents. '''

    library_collection_pattern = re.compile(
        r'\(asset_type:question OR asset_type:block\) AND '
        r'\(parent__uid:([^)]+)\)'
    )

    def filter_queryset(self, request, queryset, view):
        if ('parent' in request.query_params and
                request.query_params['parent'] == ''):
            # Empty string means query for null parent
            queryset = queryset.filter(parent=None)
        try:
            q = request.query_params['q']
        except KeyError:
            return queryset

        # Short-circuit some commonly used queries
        COMMON_QUERY_TO_ORM_FILTER = {
            'asset_type:block': {'asset_type': 'block'},
            'asset_type:question': {'asset_type': 'question'},
            'asset_type:survey': {'asset_type': 'survey'},
            'asset_type:question OR asset_type:block': {
                'asset_type__in': ('question', 'block')
            }
        }
        try:
            return queryset.filter(**COMMON_QUERY_TO_ORM_FILTER[q])
        except KeyError:
            # We don't know how to short-circuit this query; pass it along
            pass
        except FieldError:
            # The user passed a query we recognized as commonly-used, but the
            # field was invalid for the requested model
            return queryset.none()

        # Queries for library questions/blocks inside collections are also
        # common (and buggy when using Whoosh: see #1707)
        library_collection_match = self.library_collection_pattern.match(q)
        if library_collection_match:
            try:
                return queryset.filter(
                    asset_type__in=('question', 'block'),
                    parent__uid=library_collection_match.groups()[0]
                )
            except FieldError:<commit_msg>Improve library collections filter optimization<commit_after>import re
import haystack
from distutils.util import strtobool
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import FieldError
from haystack.backends.whoosh_backend import WhooshSearchBackend
from haystack.constants import DJANGO_CT, ITERATOR_LOAD_PER_QUERY
from haystack.query import SearchQuerySet
from haystack.utils import get_model_ct
from rest_framework import filters
from whoosh.qparser import QueryParser
from whoosh.query import Term, And

from .models import Asset
from .models.object_permission import get_objects_for_user, get_anonymous_user


class AssetOwnerFilterBackend(filters.BaseFilterBackend):
    """
    For use with AssetVersions
    Restricts access to items that are owned by the current user
    """
    def filter_queryset(self, request, queryset, view):
        return queryset.filter(asset__owner=request.user)


class KpiObjectPermissionsFilter(object):
    perm_format = '%(app_label)s.view_%(model_name)s'

    def filter_queryset(self, request, queryset, view):
        user = request.user
        if user.is_superuser and view.action != 'list':
            # For a list, we won't deluge the superuser with everyone else's
            # stuff. This isn't a list, though, so return it all
            return queryset
        # Governs whether unsubscribed (but publicly discoverable) objects are
        # included. Exclude them by default
        all_public = bool(strtobool(
            request.query_params.get('all_public', 'false').lower()))

        model_cls = queryset.model
        kwargs = {
            'app_label': model_cls._meta.app_label,
            'model_name': model_cls._meta.model_name,
        }
        permission = self.perm_format % kwargs

        if user.is_anonymous():
            user = get_anonymous_user()
            # Avoid giving anonymous users special treatment when viewing
            # public objects
            owned_and_explicitly_shared = queryset.none()
        else:
            owned_and_explicitly_shared = get_objects_for_user(
                user, permission, queryset)
        public = get_objects_for_user(
            get_anonymous_user(), permission, queryset)
        if view.action != 'list':
            # Not a list, so discoverability doesn't matter
            return owned_and_explicitly_shared | public

        # For a list, do not include public objects unless they are also
        # discoverable
        try:
            discoverable = public.filter(discoverable_when_public=True)
        except FieldError:
            try:
                # The model does not have a discoverability setting, but maybe
                # its parent does
                discoverable = public.filter(
                    parent__discoverable_when_public=True)
            except FieldError:
                # Neither the model or its parent has a discoverability setting
                discoverable = public.none()

        if all_public:
            # We were asked not to consider subscriptions; return all
            # discoverable objects
            return owned_and_explicitly_shared | discoverable

        # Of the discoverable objects, determine to which the user has
        # subscribed
        try:
            subscribed = public.filter(usercollectionsubscription__user=user)
        except FieldError:
            try:
                # The model does not have a subscription relation, but maybe
                # its parent does
                subscribed = public.filter(
                    parent__usercollectionsubscription__user=user)
            except FieldError:
                # Neither the model or its parent has a subscription relation
                subscribed = public.none()

        return owned_and_explicitly_shared | subscribed


class RelatedAssetPermissionsFilter(KpiObjectPermissionsFilter):
    ''' Uses KpiObjectPermissionsFilter to determine which assets the user
    may access, and then filters the provided queryset to include only objects
    related to those assets. The queryset's model must be related to `Asset`
    via a field named `asset`. '''

    def filter_queryset(self, request, queryset, view):
        available_assets = super(
            RelatedAssetPermissionsFilter, self
        ).filter_queryset(
            request=request,
            queryset=Asset.objects.all(),
            view=view
        )
        return queryset.filter(asset__in=available_assets)


class SearchFilter(filters.BaseFilterBackend):
    ''' Filter objects by searching with Whoosh if the request includes a `q`
    parameter. Another parameter, `parent`, is recognized when its value is an
    empty string; this restricts the queryset to objects without parents. '''

    library_collection_pattern = re.compile(
        r'\(((?:asset_type:(?:[^ ]+)(?: OR )*)+)\) AND \(parent__uid:([^)]+)\)'
    )

    def filter_queryset(self, request, queryset, view):
        if ('parent' in request.query_params and
                request.query_params['parent'] == ''):
            # Empty string means query for null parent
            queryset = queryset.filter(parent=None)
        try:
            q = request.query_params['q']
        except KeyError:
            return queryset

        # Short-circuit some commonly used queries
        COMMON_QUERY_TO_ORM_FILTER = {
            'asset_type:block': {'asset_type': 'block'},
            'asset_type:question': {'asset_type': 'question'},
            'asset_type:survey': {'asset_type': 'survey'},
            'asset_type:question OR asset_type:block': {
                'asset_type__in': ('question', 'block')
            }
        }
        try:
            return queryset.filter(**COMMON_QUERY_TO_ORM_FILTER[q])
        except KeyError:
            # We don't know how to short-circuit this query; pass it along
            pass
        except FieldError:
            # The user passed a query we recognized as commonly-used, but the
            # field was invalid for the requested model
            return queryset.none()

        # Queries for library questions/blocks inside collections are also
        # common (and buggy when using Whoosh: see #1707)
        library_collection_match = self.library_collection_pattern.match(q)
        if library_collection_match:
            asset_types = [
                type_query.split(':')[1] for type_query in
                    library_collection_match.groups()[0].split(' OR ')
            ]
            parent__uid = library_collection_match.groups()[1]
            try:
                return queryset.filter(
                    asset_type__in=asset_types,
                    parent__uid=parent__uid
                )
            except FieldError:
{'fraction_non_alphanumeric': 0.06953667089457584, 'fraction_numerical': 0.0008953219428486159, 'mean_word_length': 2.3754721732561066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5345229', 'n_tokens_mistral': 3426, 'n_tokens_neox': 3344, 'n_words': 1040}