<commit_before>import os
from django.db.models.fields.files import FieldFile
import six

from django import forms
from django.core.files.base import ContentFile
from django.db import models
from django.db.models import get_model
from django.utils.translation import ungettext

from tgmfiles.forms import allowed_type
from tgmfiles.models import get_upload_path, TemporaryFileWrapper, human_readable_types
from tgmfiles.widgets import TgmSingleUploadWidget, TgmMultiUploadWidget


class TgmFormFileField(forms.FileField):
    def __init__(self, allowed_types, fq, widget, *args, **kwargs):
        self.field_query = fq
        self.allowed_types = allowed_types
        self.widget = widget(fq=self.field_query, is_image=False)

        super(TgmFormFileField, self).__init__(*args, **kwargs)

    @staticmethod
    def get_content_type(the_file):
        try:
            temporary = TemporaryFileWrapper.objects.get(file=the_file)
            return temporary.content_type

        except TemporaryFileWrapper.DoesNotExist:
            return 'application/unknown'

    @staticmethod
    def file_type_error(content_type, allowed_types):
        raise forms.ValidationError(ungettext("File should be a %s.",
                                              "File should be one of the following types [%s]",
                                              len(allowed_types)) % (human_readable_types(allowed_types)))

    def to_python(self, data):
        if isinstance(data, six.string_types) and data[:3] == 'id:':
            # Pre uploaded linked file.
            return TemporaryFileWrapper.get_image_from_id(data[3:], self.field_query)

        data = super(TgmFormFileField, self).to_python(data)

        if data:
            content_type = TgmFormFileField.get_content_type(data)
            if not allowed_type(content_type, self.allowed_types):
                self.file_type_error(content_type, self.allowed_types)

        return data


class TgmFormImageField(forms.ImageField):
    def __init__(self, allowed_types, fq, widget, *args, **kwargs):
        self.field_query = fq
        self.allowed_types = allowed_types
        self.widget = widget(fq=self.field_query, is_image=True)

        super(TgmFormImageField, self).__init__(*args, **kwargs)

    def to_python(self, data):
        if isinstance(data, six.string_types) and data[:3] == 'id:':
            # Pre uploaded linked file.
            return TemporaryFileWrapper.get_image_from_id(data[3:], self.field_query)

        data = super(TgmFormImageField, self).to_python(data)
        if data:
            content_type = TgmFormFileField.get_content_type(data)
            if not allowed_type(content_type, self.allowed_types):
                TgmFormFileField.file_type_error(content_type, self.allowed_types)

        return data


class TgmFileField(models.FileField):
    DEFAULT_FILE_TYPES = ['application/pdf', 'application/x-rar-compressed', 'application/zip']

    def __init__(self, post_link=None, allowed_types=None, widget=None, get_upload_image=None, **kwargs):
        self.widget = widget or self.get_widget_class()
        self.field_query = None
        self.max_length = 128
        self.get_upload_image = get_upload_image

        # Used for validators
        self.allowed_types = self.handle_allowed_types(allowed_types or self.DEFAULT_FILE_TYPES)

        if post_link is not None and callable(post_link):
            setattr(self, 'post_link', post_link)

        super(TgmFileField, self).__init__(**kwargs)

    def contribute_to_class(self, cls, name):
        self.field_query = [cls, name]

        super(TgmFileField, self).contribute_to_class(cls, name)

<commit_msg>Post link type validation is not needed since files are validated when linking.<commit_after>import os
from django.db.models.fields.files import FieldFile
import six

from django import forms
from django.core.files.base import ContentFile
from django.db import models
from django.db.models import get_model
from django.utils.translation import ungettext

from tgmfiles.forms import allowed_type
from tgmfiles.models import get_upload_path, TemporaryFileWrapper, human_readable_types
from tgmfiles.widgets import TgmSingleUploadWidget, TgmMultiUploadWidget


class TgmFormFileField(forms.FileField):
    def __init__(self, allowed_types, fq, widget, *args, **kwargs):
        self.field_query = fq
        self.allowed_types = allowed_types
        self.widget = widget(fq=self.field_query, is_image=False)

        super(TgmFormFileField, self).__init__(*args, **kwargs)

    @staticmethod
    def get_content_type(the_file):
        try:
            temporary = TemporaryFileWrapper.objects.get(file=the_file)
            return temporary.content_type

        except TemporaryFileWrapper.DoesNotExist:
            return None

    @staticmethod
    def file_type_error(content_type, allowed_types):
        raise forms.ValidationError(ungettext("File should be a %s.",
                                              "File should be one of the following types [%s]",
                                              len(allowed_types)) % (human_readable_types(allowed_types)))

    def to_python(self, data):
        if isinstance(data, six.string_types) and data[:3] == 'id:':
            # Pre uploaded linked file.
            return TemporaryFileWrapper.get_image_from_id(data[3:], self.field_query)

        data = super(TgmFormFileField, self).to_python(data)

        if data:
            content_type = TgmFormFileField.get_content_type(data)
            if content_type is not None and not allowed_type(content_type, self.allowed_types):
                self.file_type_error(content_type, self.allowed_types)

        return data


class TgmFormImageField(forms.ImageField):
    def __init__(self, allowed_types, fq, widget, *args, **kwargs):
        self.field_query = fq
        self.allowed_types = allowed_types
        self.widget = widget(fq=self.field_query, is_image=True)

        super(TgmFormImageField, self).__init__(*args, **kwargs)

    def to_python(self, data):
        if isinstance(data, six.string_types) and data[:3] == 'id:':
            # Pre uploaded linked file.
            return TemporaryFileWrapper.get_image_from_id(data[3:], self.field_query)

        data = super(TgmFormImageField, self).to_python(data)
        if data:
            content_type = TgmFormFileField.get_content_type(data)
            if content_type is not None and not allowed_type(content_type, self.allowed_types):
                TgmFormFileField.file_type_error(content_type, self.allowed_types)

        return data


class TgmFileField(models.FileField):
    DEFAULT_FILE_TYPES = ['application/pdf', 'application/x-rar-compressed', 'application/zip']

    def __init__(self, post_link=None, allowed_types=None, widget=None, get_upload_image=None, **kwargs):
        self.widget = widget or self.get_widget_class()
        self.field_query = None
        self.max_length = 128
        self.get_upload_image = get_upload_image

        # Used for validators
        self.allowed_types = self.handle_allowed_types(allowed_types or self.DEFAULT_FILE_TYPES)

        if post_link is not None and callable(post_link):
            setattr(self, 'post_link', post_link)

        super(TgmFileField, self).__init__(**kwargs)

    def contribute_to_class(self, cls, name):
        self.field_query = [cls, name]

        super(TgmFileField, self).contribute_to_class(cls, name)


{'fraction_non_alphanumeric': 0.08665325285043594, 'fraction_numerical': 0.0018779342723004694, 'mean_word_length': 3.12617598229109, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3054648', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2143, 'n_words': 478}