<commit_before>__version__ = (1, 0, 0, "final", 0)

from django.db import models
from django import forms
from django.template.defaultfilters import filesizeformat
from django.utils.translation import ugettext as _

import magic


class ValidatedFileField(models.FileField):
    def __init__(self, *args, **kwargs):
        self.content_types = kwargs.pop("content_types", [])
        self.max_upload_size = kwargs.pop("max_upload_size", 0)
        super(ValidatedFileField, self).__init__(*args, **kwargs)

    def clean(self, *args, **kwargs):
        data = super(ValidatedFileField, self).clean(*args, **kwargs)
        file = data.file

        if self.content_types:
            uploaded_content_type = getattr(file, 'content_type', '')

            mg = magic.Magic(mime=True)
            content_type_magic = mg.from_buffer(file.read())
            file.seek(0)

            # Prefere mime-type instead mime-type from http header
            if uploaded_content_type != content_type_magic:
                uploaded_content_type = content_type_magic

            if not uploaded_content_type in self.content_types:
                raise forms.ValidationError(_('Files of type %(type)s are not supported.') % {'type': content_type_magic})

        if self.max_upload_size and hasattr(file, '_size'):
            if file._size > self.max_upload_size:
                raise forms.ValidationError(_('Files of size greater than %(max_size)s are not allowed. Your file is %(current_size)s') %
                                            {'max_size': filesizeformat(self.max_upload_size),
                                             'current_size': filesizeformat(file._size)})

        return data


class FileQuota(object):

    def __init__(self, max_usage = -1):
        self.current_usage = 0
        self.max_usage = max_usage

    def update(self, items, attr_name):
        self.current_usage = 0
        for item in items:
            the_file = getattr(item, attr_name, None)
            if the_file:
                try:<commit_msg>Replaced hard coded mime lookup length with kwarg variable<commit_after>__version__ = (1, 0, 0, "final", 0)

from django.db import models
from django import forms
from django.template.defaultfilters import filesizeformat
from django.utils.translation import ugettext as _

import magic


class ValidatedFileField(models.FileField):
    def __init__(self, *args, **kwargs):
        self.content_types = kwargs.pop("content_types", [])
        self.max_upload_size = kwargs.pop("max_upload_size", 0)
        self.mime_lookup_length = kwargs.pop("mime_lookup_length", 4096)
        super(ValidatedFileField, self).__init__(*args, **kwargs)

    def clean(self, *args, **kwargs):
        data = super(ValidatedFileField, self).clean(*args, **kwargs)
        file = data.file

        if self.content_types:
            uploaded_content_type = getattr(file, 'content_type', '')

            mg = magic.Magic(mime=True)
            content_type_magic = mg.from_buffer(
                file.read(self.mime_lookup_length)
            )
            file.seek(0)

            # Prefere mime-type instead mime-type from http header
            if uploaded_content_type != content_type_magic:
                uploaded_content_type = content_type_magic

            if not uploaded_content_type in self.content_types:
                raise forms.ValidationError(_('Files of type %(type)s are not supported.') % {'type': content_type_magic})

        if self.max_upload_size and hasattr(file, '_size'):
            if file._size > self.max_upload_size:
                raise forms.ValidationError(_('Files of size greater than %(max_size)s are not allowed. Your file is %(current_size)s') %
                                            {'max_size': filesizeformat(self.max_upload_size),
                                             'current_size': filesizeformat(file._size)})

        return data


class FileQuota(object):

    def __init__(self, max_usage = -1):
        self.current_usage = 0
        self.max_usage = max_usage

    def update(self, items, attr_name):
        self.current_usage = 0
        for item in items:
            the_file = getattr(item, attr_name, None)
            if the_file:
                try:
{'fraction_non_alphanumeric': 0.09556233857713078, 'fraction_numerical': 0.005165531814980042, 'mean_word_length': 2.418940609951846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5869110', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1176, 'n_words': 293}