<commit_before>import anyjson
from email import Encoders
from email.header import Header
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
from email.mime.text import MIMEText
import logging
import mimetypes
import os
import textwrap

from django.conf import settings
from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
from django.core.files.storage import default_storage
from django.core.mail import SafeMIMEText, SafeMIMEMultipart
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models.signals import post_delete
from django.dispatch import receiver
from django_extensions.db.models import TimeStampedModel
from django.utils.translation import ugettext_lazy as _
from oauth2client.contrib.django_orm import CredentialsField

from lily.tenant.models import TenantMixin
from lily.users.models import LilyUser
from lily.utils.models.mixins import DeletedMixin

from ..sanitize import sanitize_html_email


logger = logging.getLogger(__name__)


def get_attachment_upload_path(instance, filename):
    if isinstance(instance, EmailOutboxAttachment):
        message_id = instance.email_outbox_message_id
    else:
        message_id = instance.message_id

    return settings.EMAIL_ATTACHMENT_UPLOAD_TO % {
        'tenant_id': instance.tenant_id,
        'message_id': message_id,
        'filename': filename
    }


def get_template_attachment_upload_path(instance, filename):
    return settings.EMAIL_TEMPLATE_ATTACHMENT_UPLOAD_TO % {
        'tenant_id': instance.tenant_id,
        'template_id': instance.template_id,
        'filename': filename
    }


def get_outbox_attachment_upload_path(instance, filename):
    return settings.EMAIL_ATTACHMENT_UPLOAD_TO % {
        'tenant_id': instance.tenant_id,
        'message_id': instance.email_outbox_message_id,
        'filename': filename
    }


class EmailAccount(TenantMixin, DeletedMixin):
    """
    Email Account linked to a user.
    """
    email_address = models.EmailField(max_length=254)
    from_name = models.CharField(max_length=254, default='')
    label = models.CharField(max_length=254, default='')
    is_authorized = models.BooleanField(default=False)

    # History id is a field to keep track of the sync status of a gmail box.
    history_id = models.BigIntegerField(null=True)
    temp_history_id = models.BigIntegerField(null=True)
    first_sync_finished = models.BooleanField(default=False)

    owner = models.ForeignKey(LilyUser, related_name='email_accounts_owned')
    shared_with_users = models.ManyToManyField(
        LilyUser,
        related_name='shared_email_accounts',
        help_text=_('Select the users wich to share the account with.'),
        blank=True,
    )
    public = models.BooleanField(
        default=False,
        help_text=_('Make the email account accessible for the whole company.')
    )

    def __unicode__(self):
        return u'%s  (%s)' % (self.label, self.email_address)

    def is_owned_by_user(self):
        return True

    class Meta:
        app_label = 'email'


class SharedEmailConfig(TenantMixin):
    """
    Settings that a user can set on email accounts shared with this user.
    """
    email_account = models.ForeignKey(EmailAccount)
    user = models.ForeignKey(LilyUser)
    is_hidden = models.BooleanField(default=False)

    class Meta:
        app_label = 'email'
        unique_together = ('tenant', 'email_account', 'user')


class GmailCredentialsModel(models.Model):
    """
    OAuth2 credentials for gmail api
    """
    id = models.ForeignKey(EmailAccount, primary_key=True)
    credentials = CredentialsField()

    class Meta:
        app_label = 'email'


class EmailLabel(models.Model):
    """
    Label for EmailAccount and EmailMessage
    """
    LABEL_SYSTEM, LABEL_USER = range(2)
    LABEL_TYPES = (
        (LABEL_SYSTEM, _('System')),
        (LABEL_USER, _('User')),
    )

    account = models.ForeignKey(EmailAccount, related_name='labels')
    label_type = models.IntegerField(choices=LABEL_TYPES, default=LABEL_SYSTEM)
    label_id = models.CharField(max_length=255)
    name = models.CharField(max_length=255)
    unread = models.PositiveIntegerField(default=0)  # Number of unread email messages with this label.

    def __unicode__(self):
        return self.name

    class Meta:
        app_label = 'email'
        unique_together = ('account', 'label_id')


class Recipient(models.Model):
    """
    Name and email address of a recipient
    """
    name = models.CharField(max_length=1000, null=True)
    email_address = models.CharField(max_length=1000, null=True, db_index=True)

    def __unicode__(self):
        return u'%s <%s>' % (self.name, self.email_address)

    class Meta:
        app_label = 'email'
        unique_together = ('name', 'email_address')


class EmailMessage(models.Model):
    """
    EmailMessage has all information from an email message
    """
    account = models.ForeignKey(EmailAccount, related_name='messages')
    body_html = models.TextField(default='')
    body_text = models.TextField(default='')
    draft_id = models.CharField(max_length=50, db_index=True, default='')
    has_attachment = models.BooleanField(default=False)
    labels = models.ManyToManyField(EmailLabel, related_name='messages')
    message_id = models.CharField(max_length=50, db_index=True)
    read = models.BooleanField(default=False, db_index=True)
    received_by = models.ManyToManyField(Recipient, related_name='received_messages')
    received_by_cc = models.ManyToManyField(Recipient, related_name='received_messages_as_cc')
    sender = models.ForeignKey(Recipient, related_name='sent_messages')
    sent_date = models.DateTimeField(db_index=True)
    snippet = models.TextField(default='')
    subject = models.TextField(default='')
    thread_id = models.CharField(max_length=50, db_index=True)

    @property
    def tenant_id(self):
        return self.account.tenant_id

    @property
    def reply_body(self):
        from ..utils import create_a_beautiful_soup_object
        """
        Return a version of the body which is used for replies or forwards.
        This is preferably the html part, but in case that doesn't exist we use the plain text part.
        """
        if self.body_html:
            # In case of html, wrap body in blockquote tag.
            soup = create_a_beautiful_soup_object(self.body_html)
            if not soup.html:
                # Haven't figured out yet how to do this elegantly.
                html = '<html>%s</html>' % self.body_html
                soup = create_a_beautiful_soup_object(html)

            if not soup or soup.get_text == "":
                html = self.body_html
            else:
                soup.html.unwrap()
                html = soup.decode()

            html = sanitize_html_email(html)
            return html
        elif self.body_text:
            # In case of plain text, prepend '>' to every line of body.
            reply_body = []
            for line in self.body_text.splitlines():
                reply_body.extend(textwrap.wrap(line, 120))
            reply_body = ['> %s' % line for line in reply_body]
            return '<br /><br /><br />'.join(reply_body)
        else:
            return ''

    def fast_label_check(self, label_name):
        """
        Do a label check that checks if a field is prefetched, making it way
        faster for optimized queries.
        """
        if hasattr(self, '_prefetched_objects_cache') and 'labels' in self._prefetched_objects_cache:
            return label_name in [label.label_id for label in self.labels.all()]
        else:
            return self.labels.filter(label_id=label_name).exists()

    @property
    def is_trashed(self):
        # When the instance variable is present, don't evaluate the corresponding label.
        if hasattr(self, '_is_trashed'):
            return self._is_trashed
        else:
            return self.fast_label_check(settings.GMAIL_LABEL_TRASH)

    @property
    def is_starred(self):
        # When the instance variable is present, don't evaluate the corresponding label.
        if hasattr(self, '_is_starred'):
            return self._is_starred
        else:
            return self.fast_label_check(settings.GMAIL_LABEL_STAR)

    @property
    def is_draft(self):
        return self.fast_label_check(settings.GMAIL_LABEL_DRAFT)

    @property
    def is_important(self):
        return self.fast_label_check(settings.GMAIL_LABEL_IMPORTANT)

    @property
    def is_spam(self):
        # When the instance variable is present, don't evaluate the corresponding label.
        if hasattr(self, '_is_spam'):
            return self._is_spam
        else:
            return self.fast_label_check(settings.GMAIL_LABEL_SPAM)

    @property
    def is_archived(self):
        # When the instance variable is present, don't evaluate the corresponding label.
        if hasattr(self, '_is_archived'):
            return self._is_archived
        else:
            return not self.fast_label_check(settings.GMAIL_LABEL_INBOX)

    def get_message_id(self):
        header = self.headers.filter(name__istartswith='message-id').first()
        if header:
            return header.value

    @property
    def reply_to(self):
        """
        Return the reply to address if it is present as a header, otherwise use email address of the sender.
        """
        header = self.headers.filter(name__istartswith='reply-to').first()
        if header:
            return header.value
        else:
            return self.sender.email_address

    def __unicode__(self):
        return u'%s: %s' % (self.sender, self.snippet)

    class Meta:
        app_label = 'email'
        unique_together = ('account', 'message_id')<commit_msg>LILY-1935: extract email address from reply-to header.<commit_after>import anyjson
from email import Encoders
from email.header import Header
from email.mime.audio import MIMEAudio
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
from email.mime.text import MIMEText
from email.utils import parseaddr
import logging
import mimetypes
import os
import textwrap

from django.conf import settings
from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
from django.core.files.storage import default_storage
from django.core.mail import SafeMIMEText, SafeMIMEMultipart
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models.signals import post_delete
from django.dispatch import receiver
from django_extensions.db.models import TimeStampedModel
from django.utils.translation import ugettext_lazy as _
from oauth2client.contrib.django_orm import CredentialsField

from lily.tenant.models import TenantMixin
from lily.users.models import LilyUser
from lily.utils.models.mixins import DeletedMixin

from ..sanitize import sanitize_html_email


logger = logging.getLogger(__name__)


def get_attachment_upload_path(instance, filename):
    if isinstance(instance, EmailOutboxAttachment):
        message_id = instance.email_outbox_message_id
    else:
        message_id = instance.message_id

    return settings.EMAIL_ATTACHMENT_UPLOAD_TO % {
        'tenant_id': instance.tenant_id,
        'message_id': message_id,
        'filename': filename
    }


def get_template_attachment_upload_path(instance, filename):
    return settings.EMAIL_TEMPLATE_ATTACHMENT_UPLOAD_TO % {
        'tenant_id': instance.tenant_id,
        'template_id': instance.template_id,
        'filename': filename
    }


def get_outbox_attachment_upload_path(instance, filename):
    return settings.EMAIL_ATTACHMENT_UPLOAD_TO % {
        'tenant_id': instance.tenant_id,
        'message_id': instance.email_outbox_message_id,
        'filename': filename
    }


class EmailAccount(TenantMixin, DeletedMixin):
    """
    Email Account linked to a user.
    """
    email_address = models.EmailField(max_length=254)
    from_name = models.CharField(max_length=254, default='')
    label = models.CharField(max_length=254, default='')
    is_authorized = models.BooleanField(default=False)

    # History id is a field to keep track of the sync status of a gmail box.
    history_id = models.BigIntegerField(null=True)
    temp_history_id = models.BigIntegerField(null=True)
    first_sync_finished = models.BooleanField(default=False)

    owner = models.ForeignKey(LilyUser, related_name='email_accounts_owned')
    shared_with_users = models.ManyToManyField(
        LilyUser,
        related_name='shared_email_accounts',
        help_text=_('Select the users wich to share the account with.'),
        blank=True,
    )
    public = models.BooleanField(
        default=False,
        help_text=_('Make the email account accessible for the whole company.')
    )

    def __unicode__(self):
        return u'%s  (%s)' % (self.label, self.email_address)

    def is_owned_by_user(self):
        return True

    class Meta:
        app_label = 'email'


class SharedEmailConfig(TenantMixin):
    """
    Settings that a user can set on email accounts shared with this user.
    """
    email_account = models.ForeignKey(EmailAccount)
    user = models.ForeignKey(LilyUser)
    is_hidden = models.BooleanField(default=False)

    class Meta:
        app_label = 'email'
        unique_together = ('tenant', 'email_account', 'user')


class GmailCredentialsModel(models.Model):
    """
    OAuth2 credentials for gmail api
    """
    id = models.ForeignKey(EmailAccount, primary_key=True)
    credentials = CredentialsField()

    class Meta:
        app_label = 'email'


class EmailLabel(models.Model):
    """
    Label for EmailAccount and EmailMessage
    """
    LABEL_SYSTEM, LABEL_USER = range(2)
    LABEL_TYPES = (
        (LABEL_SYSTEM, _('System')),
        (LABEL_USER, _('User')),
    )

    account = models.ForeignKey(EmailAccount, related_name='labels')
    label_type = models.IntegerField(choices=LABEL_TYPES, default=LABEL_SYSTEM)
    label_id = models.CharField(max_length=255)
    name = models.CharField(max_length=255)
    unread = models.PositiveIntegerField(default=0)  # Number of unread email messages with this label.

    def __unicode__(self):
        return self.name

    class Meta:
        app_label = 'email'
        unique_together = ('account', 'label_id')


class Recipient(models.Model):
    """
    Name and email address of a recipient
    """
    name = models.CharField(max_length=1000, null=True)
    email_address = models.CharField(max_length=1000, null=True, db_index=True)

    def __unicode__(self):
        return u'%s <%s>' % (self.name, self.email_address)

    class Meta:
        app_label = 'email'
        unique_together = ('name', 'email_address')


class EmailMessage(models.Model):
    """
    EmailMessage has all information from an email message
    """
    account = models.ForeignKey(EmailAccount, related_name='messages')
    body_html = models.TextField(default='')
    body_text = models.TextField(default='')
    draft_id = models.CharField(max_length=50, db_index=True, default='')
    has_attachment = models.BooleanField(default=False)
    labels = models.ManyToManyField(EmailLabel, related_name='messages')
    message_id = models.CharField(max_length=50, db_index=True)
    read = models.BooleanField(default=False, db_index=True)
    received_by = models.ManyToManyField(Recipient, related_name='received_messages')
    received_by_cc = models.ManyToManyField(Recipient, related_name='received_messages_as_cc')
    sender = models.ForeignKey(Recipient, related_name='sent_messages')
    sent_date = models.DateTimeField(db_index=True)
    snippet = models.TextField(default='')
    subject = models.TextField(default='')
    thread_id = models.CharField(max_length=50, db_index=True)

    @property
    def tenant_id(self):
        return self.account.tenant_id

    @property
    def reply_body(self):
        from ..utils import create_a_beautiful_soup_object
        """
        Return a version of the body which is used for replies or forwards.
        This is preferably the html part, but in case that doesn't exist we use the plain text part.
        """
        if self.body_html:
            # In case of html, wrap body in blockquote tag.
            soup = create_a_beautiful_soup_object(self.body_html)
            if not soup.html:
                # Haven't figured out yet how to do this elegantly.
                html = '<html>%s</html>' % self.body_html
                soup = create_a_beautiful_soup_object(html)

            if not soup or soup.get_text == "":
                html = self.body_html
            else:
                soup.html.unwrap()
                html = soup.decode()

            html = sanitize_html_email(html)
            return html
        elif self.body_text:
            # In case of plain text, prepend '>' to every line of body.
            reply_body = []
            for line in self.body_text.splitlines():
                reply_body.extend(textwrap.wrap(line, 120))
            reply_body = ['> %s' % line for line in reply_body]
            return '<br /><br /><br />'.join(reply_body)
        else:
            return ''

    def fast_label_check(self, label_name):
        """
        Do a label check that checks if a field is prefetched, making it way
        faster for optimized queries.
        """
        if hasattr(self, '_prefetched_objects_cache') and 'labels' in self._prefetched_objects_cache:
            return label_name in [label.label_id for label in self.labels.all()]
        else:
            return self.labels.filter(label_id=label_name).exists()

    @property
    def is_trashed(self):
        # When the instance variable is present, don't evaluate the corresponding label.
        if hasattr(self, '_is_trashed'):
            return self._is_trashed
        else:
            return self.fast_label_check(settings.GMAIL_LABEL_TRASH)

    @property
    def is_starred(self):
        # When the instance variable is present, don't evaluate the corresponding label.
        if hasattr(self, '_is_starred'):
            return self._is_starred
        else:
            return self.fast_label_check(settings.GMAIL_LABEL_STAR)

    @property
    def is_draft(self):
        return self.fast_label_check(settings.GMAIL_LABEL_DRAFT)

    @property
    def is_important(self):
        return self.fast_label_check(settings.GMAIL_LABEL_IMPORTANT)

    @property
    def is_spam(self):
        # When the instance variable is present, don't evaluate the corresponding label.
        if hasattr(self, '_is_spam'):
            return self._is_spam
        else:
            return self.fast_label_check(settings.GMAIL_LABEL_SPAM)

    @property
    def is_archived(self):
        # When the instance variable is present, don't evaluate the corresponding label.
        if hasattr(self, '_is_archived'):
            return self._is_archived
        else:
            return not self.fast_label_check(settings.GMAIL_LABEL_INBOX)

    def get_message_id(self):
        header = self.headers.filter(name__istartswith='message-id').first()
        if header:
            return header.value

    @property
    def reply_to(self):
        """
        Return the reply to address if it is present as a header, otherwise use email address of the sender.
        """
        header = self.headers.filter(name__istartswith='reply-to').first()
        if header:
            # A reply-to header can contain a plain email address, an email address enclosed by brackets,
            # or has a "name" <<EMAIL>> construction.
            # So return only the email address instead of the actual header value.
            return parseaddr(header.value)[1]
        else:
            return self.sender.email_address

    def __unicode__(self):
        return u'%s: %s' % (self.sender, self.snippet)

    class Meta:
        app_label = 'email'
        unique_together = ('account', 'message_id')
{'fraction_non_alphanumeric': 0.08823236349143085, 'fraction_numerical': 0.0038361897170187324, 'mean_word_length': 3.0388329979879276, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3295970', 'n_tokens_mistral': 5951, 'n_tokens_neox': 5825, 'n_words': 1460}