<commit_before>import datetime
import logging
import os
import re

from django.conf import settings
from django.utils.translation import ugettext_lazy as _, ungettext_lazy
from django.template.defaultfilters import slugify
from django.utils import timezone
from django import forms
from django.db import transaction
from django.template.loader import render_to_string

from froide.publicbody.models import PublicBody
from froide.publicbody.widgets import PublicBodySelect
from froide.helper.widgets import (
    BootstrapRadioSelect, BootstrapFileInput,
    BootstrapCheckboxInput
)
from froide.helper.text_utils import redact_subject
from froide.helper.text_diff import get_diff_chunks
from froide.upload.models import Upload

from ..models import (
    FoiMessage, FoiAttachment
)
from ..foi_mail import generate_foirequest_files
from ..tasks import convert_attachment_task, move_upload_to_attachment
from ..validators import (
    validate_upload_document, validate_postal_content_type
)
from ..utils import (
    construct_message_body, MailAttachmentSizeChecker,
    possible_reply_addresses, get_info_for_email, make_name_unique,
    redact_plaintext_with_request
)

publishing_denied = settings.FROIDE_CONFIG.get('publishing_denied', False)

ACCEPT_FILETYPES = 'image/png,image/jpeg,image/gif,application/pdf'


class AttachmentSaverMixin(object):
    def clean_files(self):
        if '%s-files' % self.prefix not in self.files:
            return self.cleaned_data['files']
        files = self.files.getlist('%s-files' % self.prefix)
        names = set()
        for file in files:
            validate_upload_document(file)
            name = self.make_filename(file.name)
            if name in names:
                # FIXME: dont make this a requirement
                raise forms.ValidationError(_('Upload files must have distinct names'))
            names.add(name)
        return self.cleaned_data['files']

    def make_filename(self, name):
        name = os.path.basename(name).rsplit('.', 1)
        return '.'.join([slugify(n) for n in name])

    def get_or_create_attachment(self, message, filename):
        try:
            att = FoiAttachment.objects.get(
                belongs_to=message,
                name=filename
            )
            return att, False
        except FoiAttachment.DoesNotExist:
            pass
        att = FoiAttachment(belongs_to=message, name=filename)
        return att, True

    def save_attachments(self, files, message, replace=False, save_file=True):
        added = []
        updated = []

        for file in files:
            filename = self.make_filename(file.name)
            if replace:
                att, created = self.get_or_create_attachment(message, filename)
            else:
                created = True
                att = FoiAttachment(belongs_to=message, name=filename)

            if created:
                added.append(att)
            else:
                updated.append(att)
            att.size = file.size
            att.filetype = file.content_type
            if save_file:
                att.file.save(filename, file)
            else:
                att.pending = True
            att.can_approve = not message.request.not_publishable
            att.approved = False
            att.save()

            if save_file and att.can_convert_to_pdf():
                transaction.on_commit(lambda: convert_attachment_task.delay(att.id))

        message._attachments = None

        return added, updated


def get_send_message_form(*args, **kwargs):
    foirequest = kwargs.pop('foirequest')
    last_message = list(foirequest.messages)[-1]
    # Translators: message reply prefix
    prefix = _('Re:')
    subject = last_message.subject
    if not subject.startswith(str(prefix)):
        subject = '{prefix} {subject}'.format(
            prefix=prefix, subject=last_message.subject
        )
    message_ready = False
    if foirequest.is_overdue() and foirequest.awaits_response():
        message_ready = True
        days = (timezone.now() - foirequest.due_date).days + 1
        message = render_to_string('foirequest/emails/overdue_reply.txt', {
            'due': ungettext_lazy(
                "%(count)s day",
                "%(count)s days",
                days) % {'count': days},
            'foirequest': foirequest
        })
    else:
        message = _("Dear Sir or Madam,\n\n...\n\nSincerely yours\n%(name)s\n")
        message = message % {'name': foirequest.user.get_full_name()}

    return SendMessageForm(
        *args,
        foirequest=foirequest,
        message_ready=message_ready,
        prefix='sendmessage',
        initial={
            "subject": subject,
            "message": message
        }
    )


def get_message_sender_form(*args, **kwargs):
    foimessage = kwargs.pop('foimessage')
    return MessagePublicBodySenderForm(*args, message=foimessage)


class MessagePublicBodySenderForm(forms.Form):
    sender = forms.ModelChoiceField(
        label=_("Sending Public Body"),
        queryset=PublicBody.objects.all(),
        widget=PublicBodySelect
    )

    def __init__(self, *args, **kwargs):
        message = kwargs.pop('message', None)
        if 'initial' not in kwargs:
            if message.sender_public_body:
                kwargs['initial'] = {'sender': message.sender_public_body}
        if 'prefix' not in kwargs:
            kwargs['prefix'] = "message-sender-%d" % message.id
        self.message = message
        super().__init__(*args, **kwargs)
        self.fields['sender'].widget.set_initial_object(message.sender_public_body)

    def save(self):
        self.message.sender_public_body = self.cleaned_data['sender']
        self.message.save()

<commit_msg>Make send message factory more flexible<commit_after>import datetime
import logging
import os
import re

from django.conf import settings
from django.utils.translation import ugettext_lazy as _, ungettext_lazy
from django.template.defaultfilters import slugify
from django.utils import timezone
from django import forms
from django.db import transaction
from django.template.loader import render_to_string

from froide.publicbody.models import PublicBody
from froide.publicbody.widgets import PublicBodySelect
from froide.helper.widgets import (
    BootstrapRadioSelect, BootstrapFileInput,
    BootstrapCheckboxInput
)
from froide.helper.text_utils import redact_subject
from froide.helper.text_diff import get_diff_chunks
from froide.upload.models import Upload

from ..models import (
    FoiMessage, FoiAttachment
)
from ..foi_mail import generate_foirequest_files
from ..tasks import convert_attachment_task, move_upload_to_attachment
from ..validators import (
    validate_upload_document, validate_postal_content_type
)
from ..utils import (
    construct_message_body, MailAttachmentSizeChecker,
    possible_reply_addresses, get_info_for_email, make_name_unique,
    redact_plaintext_with_request
)

publishing_denied = settings.FROIDE_CONFIG.get('publishing_denied', False)

ACCEPT_FILETYPES = 'image/png,image/jpeg,image/gif,application/pdf'


class AttachmentSaverMixin(object):
    def clean_files(self):
        if '%s-files' % self.prefix not in self.files:
            return self.cleaned_data['files']
        files = self.files.getlist('%s-files' % self.prefix)
        names = set()
        for file in files:
            validate_upload_document(file)
            name = self.make_filename(file.name)
            if name in names:
                # FIXME: dont make this a requirement
                raise forms.ValidationError(_('Upload files must have distinct names'))
            names.add(name)
        return self.cleaned_data['files']

    def make_filename(self, name):
        name = os.path.basename(name).rsplit('.', 1)
        return '.'.join([slugify(n) for n in name])

    def get_or_create_attachment(self, message, filename):
        try:
            att = FoiAttachment.objects.get(
                belongs_to=message,
                name=filename
            )
            return att, False
        except FoiAttachment.DoesNotExist:
            pass
        att = FoiAttachment(belongs_to=message, name=filename)
        return att, True

    def save_attachments(self, files, message, replace=False, save_file=True):
        added = []
        updated = []

        for file in files:
            filename = self.make_filename(file.name)
            if replace:
                att, created = self.get_or_create_attachment(message, filename)
            else:
                created = True
                att = FoiAttachment(belongs_to=message, name=filename)

            if created:
                added.append(att)
            else:
                updated.append(att)
            att.size = file.size
            att.filetype = file.content_type
            if save_file:
                att.file.save(filename, file)
            else:
                att.pending = True
            att.can_approve = not message.request.not_publishable
            att.approved = False
            att.save()

            if save_file and att.can_convert_to_pdf():
                transaction.on_commit(lambda: convert_attachment_task.delay(att.id))

        message._attachments = None

        return added, updated


def get_send_message_form(*args, **kwargs):
    foirequest = kwargs.pop('foirequest')
    last_message = list(foirequest.messages)[-1]
    # Translators: message reply prefix
    prefix = _('Re:')
    subject = last_message.subject
    if not subject.startswith(str(prefix)):
        subject = '{prefix} {subject}'.format(
            prefix=prefix, subject=last_message.subject
        )
    message_ready = False
    if foirequest.is_overdue() and foirequest.awaits_response():
        message_ready = True
        days = (timezone.now() - foirequest.due_date).days + 1
        message = render_to_string('foirequest/emails/overdue_reply.txt', {
            'due': ungettext_lazy(
                "%(count)s day",
                "%(count)s days",
                days) % {'count': days},
            'foirequest': foirequest
        })
    else:
        message = _("Dear Sir or Madam,\n\n...\n\nSincerely yours\n%(name)s\n")
        message = message % {'name': foirequest.user.get_full_name()}
    if 'message_ready' in kwargs:
        message_ready = kwargs.pop('message_ready')

    return SendMessageForm(
        *args,
        foirequest=foirequest,
        message_ready=message_ready,
        prefix='sendmessage',
        initial={
            "subject": subject,
            "message": message
        }, **kwargs
    )


def get_message_sender_form(*args, **kwargs):
    foimessage = kwargs.pop('foimessage')
    return MessagePublicBodySenderForm(*args, message=foimessage)


class MessagePublicBodySenderForm(forms.Form):
    sender = forms.ModelChoiceField(
        label=_("Sending Public Body"),
        queryset=PublicBody.objects.all(),
        widget=PublicBodySelect
    )

    def __init__(self, *args, **kwargs):
        message = kwargs.pop('message', None)
        if 'initial' not in kwargs:
            if message.sender_public_body:
                kwargs['initial'] = {'sender': message.sender_public_body}
        if 'prefix' not in kwargs:
            kwargs['prefix'] = "message-sender-%d" % message.id
        self.message = message
        super().__init__(*args, **kwargs)
        self.fields['sender'].widget.set_initial_object(message.sender_public_body)

    def save(self):
        self.message.sender_public_body = self.cleaned_data['sender']
        self.message.save()


{'fraction_non_alphanumeric': 0.0944922547332186, 'fraction_numerical': 0.0005163511187607573, 'mean_word_length': 2.5592649310872893, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2212314', 'n_tokens_mistral': 3315, 'n_tokens_neox': 3261, 'n_words': 722}