<commit_before>#-*- coding: utf-8 -*-
import polymorphic
import hashlib
import os
import filer
import logging
import operator

from django.contrib.auth import models as auth_models
from django.urls import reverse
from django.core.files.base import ContentFile
from django.core.exceptions import ValidationError
from django.db import (models, IntegrityError, transaction)
from django.utils.translation import ugettext_lazy as _
from filer.fields.multistorage_file import MultiStorageFileField
from filer.models import mixins
from filer.utils.cms_roles import *
from filer.utils.files import matching_file_subtypes
from filer import settings as filer_settings
from django.db.models import Count
from django.utils import timezone

from polymorphic.models import PolymorphicModel
from polymorphic.managers import PolymorphicManager
from polymorphic.query import PolymorphicQuerySet
import hashlib
import os
import filer
import logging

logger = logging.getLogger(__name__)


def silence_error_if_missing_file(exception):
    """
    Ugly way of checking in an exception describes a 'missing file'.
    """
    missing_files_errs = ('no such file', 'does not exist', )

    def find_msg_in_error(msg):
        return msg in str(exception).lower()

    if not any(map(find_msg_in_error, missing_files_errs)):
        raise exception


class FileQuerySet(PolymorphicQuerySet):

    def readonly(self, user):
        Folder = filer.models.foldermodels.Folder
        return self.filter(folder__folder_type=Folder.CORE_FOLDER)

    def find_duplicates(self, file_obj):
        return self.exclude(pk=file_obj.pk).filter(sha1=file_obj.sha1)

    def restricted(self, user):
        sites = get_sites_without_restriction_perm(user)
        if not sites:
            return self.none()
        return self.filter(
            restricted=True,
            folder__site__in=sites)

    def unrestricted(self, user):
        sites = get_sites_without_restriction_perm(user)
        if not sites:
            return self
        return self.exclude(
            restricted=True,
            folder__site__in=sites)


class FileManager(PolymorphicManager):
    queryset_class = FileQuerySet

    def find_all_duplicates(self):
        return {file_data['sha1']: file_data['count']
                for file_data in self.get_queryset().values('sha1').annotate(
                    count=Count('id')).filter(count__gt=1)}


class AliveFileManager(FileManager):
    # this is required in order to make sure that other models that are
    #   related to filer files will get an DoesNotExist exception if the file
    #   is in trash
    use_for_related_fields = True

    def get_queryset(self):
        return super(AliveFileManager, self).get_queryset().filter(
            deleted_at__isnull=True)
<commit_msg>BEN-2235: Proxy all unknown method calls to the queryset<commit_after>#-*- coding: utf-8 -*-
import polymorphic
import hashlib
import os
import filer
import logging
import operator

from django.contrib.auth import models as auth_models
from django.urls import reverse
from django.core.files.base import ContentFile
from django.core.exceptions import ValidationError
from django.db import (models, IntegrityError, transaction)
from django.utils.translation import ugettext_lazy as _
from filer.fields.multistorage_file import MultiStorageFileField
from filer.models import mixins
from filer.utils.cms_roles import *
from filer.utils.files import matching_file_subtypes
from filer import settings as filer_settings
from django.db.models import Count
from django.utils import timezone

from polymorphic.models import PolymorphicModel
from polymorphic.managers import PolymorphicManager
from polymorphic.query import PolymorphicQuerySet
import hashlib
import os
import filer
import logging

logger = logging.getLogger(__name__)


def silence_error_if_missing_file(exception):
    """
    Ugly way of checking in an exception describes a 'missing file'.
    """
    missing_files_errs = ('no such file', 'does not exist', )

    def find_msg_in_error(msg):
        return msg in str(exception).lower()

    if not any(map(find_msg_in_error, missing_files_errs)):
        raise exception


class FileQuerySet(PolymorphicQuerySet):

    def readonly(self, user):
        Folder = filer.models.foldermodels.Folder
        return self.filter(folder__folder_type=Folder.CORE_FOLDER)

    def find_duplicates(self, file_obj):
        return self.exclude(pk=file_obj.pk).filter(sha1=file_obj.sha1)

    def restricted(self, user):
        sites = get_sites_without_restriction_perm(user)
        if not sites:
            return self.none()
        return self.filter(
            restricted=True,
            folder__site__in=sites)

    def unrestricted(self, user):
        sites = get_sites_without_restriction_perm(user)
        if not sites:
            return self
        return self.exclude(
            restricted=True,
            folder__site__in=sites)


class FileManager(PolymorphicManager):
    queryset_class = FileQuerySet

    # Proxy all unknown method calls to the queryset, so that its members are
    # directly accessible as PolymorphicModel.objects.*
    # Exclude any special functions (__) from this automatic proxying.
    def __getattr__(self, name):
        if name.startswith('__'):
            return super(PolymorphicManager, self).__getattr__(self, name)
        return getattr(self.all(), name)

    def find_all_duplicates(self):
        return {file_data['sha1']: file_data['count']
                for file_data in self.get_queryset().values('sha1').annotate(
                    count=Count('id')).filter(count__gt=1)}


class AliveFileManager(FileManager):
    # this is required in order to make sure that other models that are
    #   related to filer files will get an DoesNotExist exception if the file
    #   is in trash
    use_for_related_fields = True

    def get_queryset(self):
        return super(AliveFileManager, self).get_queryset().filter(
            deleted_at__isnull=True)

{'fraction_non_alphanumeric': 0.07199866977053541, 'fraction_numerical': 0.002660458929165281, 'mean_word_length': 3.605666156202144, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1176031', 'n_tokens_mistral': 1766, 'n_tokens_neox': 1673, 'n_words': 487}