<commit_before># coding: utf-8

import sys, os
import re
import time
from datetime import datetime, timedelta
import email
import lxml
from django.core.files.uploadhandler import FileUploadHandler, StopUpload, SkipFile
from django.http import HttpResponse
from django.utils.translation import ugettext as _, ugettext_lazy
from django.conf import settings
from modoboa.lib import u2u_decode, tables, imap_utf7, parameters
from modoboa.lib.webutils import static_url, size2integer
from modoboa.lib.connections import ConnectionsManager
from modoboa.lib.email_listing import MBconnector, EmailListing
from modoboa.lib.emailutils import *

from imaputils import *
from exceptions import *

class WMtable(tables.Table):
    tableid = "emails"
    styles = "table-condensed"
    idkey = "imapid"
    select = tables.ImgColumn(
        "select", cssclass="draggable left", width="2%", 
        defvalue="%spics/grippy.png" % settings.STATIC_URL,
        header="<input type='checkbox' name='toggleselect' id='toggleselect' />"
        )
    flags = tables.ImgColumn("flags", width="4%")
    withatts = tables.ImgColumn("withatts", width="2%")
    subject = tables.Column("subject", label=ugettext_lazy("Subject"), width="50%", limit=60)
    from_ = tables.Column("from", width="20%", label=ugettext_lazy("From"), limit=30)
    date = tables.Column("date", width="15%", label=ugettext_lazy("Date"))

    cols_order = ["select", "withatts", "flags", "subject", "from_", "date"]

    def parse(self, header, value):
        try:
            return getattr(IMAPheader, "parse_%s" % header)(value)
        except AttributeError:
            return value

class IMAPheader(object):
    @staticmethod
    def parse_address(value, **kwargs):
        addr = EmailAddress(value)
        if "full" in kwargs.keys() and kwargs["full"]:
            return addr.fulladdress
        return addr.name and addr.name or addr.fulladdress

    @staticmethod
    def parse_address_list(values, **kwargs):
        lst = values.split(",")
        result = ""
        for addr in lst:
            if result != "":
                result += ", "
            result += IMAPheader.parse_address(addr, **kwargs)
        return result

    @staticmethod
    def parse_from(value, **kwargs):
        return IMAPheader.parse_address(value, **kwargs)

    @staticmethod
    def parse_to(value, **kwargs):
        return IMAPheader.parse_address_list(value, **kwargs)

    @staticmethod
    def parse_cc(value, **kwargs):
        return IMAPheader.parse_address_list(value, **kwargs)

    @staticmethod
    def parse_reply_to(value, **kwargs):
        return IMAPheader.parse_address_list(value, **kwargs)

    @staticmethod
    def parse_date(value, **kwargs):
        tmp = email.utils.parsedate_tz(value)
        if not tmp:
            return value
        ndate = datetime(*(tmp)[:7])
        now = datetime.now()
        try:
            if now - ndate > timedelta(7):
                return ndate.strftime("%d.%m.%Y %H:%M")
            return ndate.strftime("%a %H:%M")
        except ValueError:
            return value

    @staticmethod
    def parse_message_id(value, **kwargs):
        return value.strip('\n')

    @staticmethod
    def parse_subject(value, **kwargs):
        try:
            return u2u_decode.u2u_decode(value)
        except UnicodeDecodeError:
            return value

class ImapListing(EmailListing):
    tpl = "webmail/index.html"
    tbltype = WMtable
    deflocation = "INBOX/"
    defcallback = "wm_updatelisting"
    reset_wm_url = False
    
    def __init__(self, user, password, **kwargs):
        self.user = user
        self.mbc = IMAPconnector(user=user.username, password=password)
        if kwargs.has_key("pattern"):
            self.parse_search_parameters(kwargs["criteria"],
                                         kwargs["pattern"])
        else:
            self.mbc.criterions = []

        super(ImapListing, self).__init__(**kwargs)
        self.extravars["refreshrate"] = \
            int(parameters.get_user(user, "REFRESH_INTERVAL")) * 1000

    def parse_search_parameters(self, criterion, pattern):
        def or_criterion(old, c):
            if old == "":
                return c
            return "OR (%s) (%s)" % (old, c)
        if criterion == "both":
            criterion = "from_addr, subject"
        criterions = ""
        for c in criterion.split(','):
            if c == "from_addr":
                key = "FROM"
            elif c == "subject":
                key = "SUBJECT"
            else:
                continue
            criterions = \
                unicode(or_criterion(criterions, '%s "%s"' % (key, pattern)))
        self.mbc.criterions = [unicode("(%s)" % criterions)]

    @staticmethod
    def computequota(mbc):
        try:
            return int(float(mbc.quota_actual) \
                           / float(mbc.quota_limit) * 100)
        except (AttributeError, TypeError):
            return -1

    def getquota(self):
        return ImapListing.computequota(self.mbc)

class ImapEmail(Email):
    headernames = [
        ('From', True),
        ('To', True),
        ('Cc', True),
        ('Date', True),
        ('Subject', True),
        ]

    def __init__(self, mbox, mailid, request, dformat="DISPLAYMODE", addrfull=False, 
                 links=0):
        mformat = parameters.get_user(request.user, "DISPLAYMODE")
        self.dformat = parameters.get_user(request.user, dformat)

        self.headers = []
        self.attachments = {}
        self.imapc = get_imapconnector(request)
        msg = self.imapc.fetchmail(mbox, mailid, readonly=False,
                                   headers=self.headers_as_list)
        self.mbox = mbox
        self.mailid = mailid
        headers = msg['BODY[HEADER.FIELDS (%s)]' % self.headers_as_text]
        fallback_fmt = "html" if self.dformat == "plain" else "plain"
        
        self.bs = BodyStructure(msg['BODYSTRUCTURE'])
        data = None

        mformat = self.dformat if self.bs.contents.has_key(self.dformat) else fallback_fmt

        if len(self.bs.contents):
            bodyc = u''
            for part in self.bs.contents[mformat]:
                pnum = part['pnum']
                data = self.imapc._cmd("FETCH", mailid, "(BODY.PEEK[%s])" % pnum)
                content = decode_payload(part['encoding'],
                                         data[int(mailid)]['BODY[%s]' % pnum])
                charset = self._find_content_charset(part)
                if charset is not None:
                    try:
                        content = content.decode(charset)
                    except (UnicodeDecodeError, LookupError):
                        import chardet
                        result = chardet.detect(content)
                        content = content.decode(result['encoding'])
                bodyc += content

            self._fetch_inlines()
            self.body = \
                getattr(self, "viewmail_%s" % mformat)(bodyc, links=links)
        else:<commit_msg>Try to convert subject using the appropriate charset.<commit_after># coding: utf-8

import sys, os
import re
import time
from datetime import datetime, timedelta
import email
import lxml
import chardet
from django.core.files.uploadhandler import FileUploadHandler, StopUpload, SkipFile
from django.http import HttpResponse
from django.utils.translation import ugettext as _, ugettext_lazy
from django.conf import settings
from modoboa.lib import u2u_decode, tables, imap_utf7, parameters
from modoboa.lib.webutils import static_url, size2integer
from modoboa.lib.connections import ConnectionsManager
from modoboa.lib.email_listing import MBconnector, EmailListing
from modoboa.lib.emailutils import *

from imaputils import *
from exceptions import *

class WMtable(tables.Table):
    tableid = "emails"
    styles = "table-condensed"
    idkey = "imapid"
    select = tables.ImgColumn(
        "select", cssclass="draggable left", width="2%", 
        defvalue="%spics/grippy.png" % settings.STATIC_URL,
        header="<input type='checkbox' name='toggleselect' id='toggleselect' />"
        )
    flags = tables.ImgColumn("flags", width="4%")
    withatts = tables.ImgColumn("withatts", width="2%")
    subject = tables.Column("subject", label=ugettext_lazy("Subject"), width="50%", limit=60)
    from_ = tables.Column("from", width="20%", label=ugettext_lazy("From"), limit=30)
    date = tables.Column("date", width="15%", label=ugettext_lazy("Date"))

    cols_order = ["select", "withatts", "flags", "subject", "from_", "date"]

    def parse(self, header, value):
        res = chardet.detect(value)
        try:
            value = getattr(IMAPheader, "parse_%s" % header)(value)
        except AttributeError:
            pass
        if not value or type(value) is unicode or res["encoding"] == "ascii":
            return value
        return value.decode(res["encoding"])

class IMAPheader(object):
    @staticmethod
    def parse_address(value, **kwargs):
        addr = EmailAddress(value)
        if "full" in kwargs.keys() and kwargs["full"]:
            return addr.fulladdress
        return addr.name and addr.name or addr.fulladdress

    @staticmethod
    def parse_address_list(values, **kwargs):
        lst = values.split(",")
        result = ""
        for addr in lst:
            if result != "":
                result += ", "
            result += IMAPheader.parse_address(addr, **kwargs)
        return result

    @staticmethod
    def parse_from(value, **kwargs):
        return IMAPheader.parse_address(value, **kwargs)

    @staticmethod
    def parse_to(value, **kwargs):
        return IMAPheader.parse_address_list(value, **kwargs)

    @staticmethod
    def parse_cc(value, **kwargs):
        return IMAPheader.parse_address_list(value, **kwargs)

    @staticmethod
    def parse_reply_to(value, **kwargs):
        return IMAPheader.parse_address_list(value, **kwargs)

    @staticmethod
    def parse_date(value, **kwargs):
        tmp = email.utils.parsedate_tz(value)
        if not tmp:
            return value
        ndate = datetime(*(tmp)[:7])
        now = datetime.now()
        try:
            if now - ndate > timedelta(7):
                return ndate.strftime("%d.%m.%Y %H:%M")
            return ndate.strftime("%a %H:%M")
        except ValueError:
            return value

    @staticmethod
    def parse_message_id(value, **kwargs):
        return value.strip('\n')

    @staticmethod
    def parse_subject(value, **kwargs):
        try:
            return u2u_decode.u2u_decode(value)
        except UnicodeDecodeError:
            return value

class ImapListing(EmailListing):
    tpl = "webmail/index.html"
    tbltype = WMtable
    deflocation = "INBOX/"
    defcallback = "wm_updatelisting"
    reset_wm_url = False
    
    def __init__(self, user, password, **kwargs):
        self.user = user
        self.mbc = IMAPconnector(user=user.username, password=password)
        if kwargs.has_key("pattern"):
            self.parse_search_parameters(kwargs["criteria"],
                                         kwargs["pattern"])
        else:
            self.mbc.criterions = []

        super(ImapListing, self).__init__(**kwargs)
        self.extravars["refreshrate"] = \
            int(parameters.get_user(user, "REFRESH_INTERVAL")) * 1000

    def parse_search_parameters(self, criterion, pattern):
        def or_criterion(old, c):
            if old == "":
                return c
            return "OR (%s) (%s)" % (old, c)
        if criterion == "both":
            criterion = "from_addr, subject"
        criterions = ""
        for c in criterion.split(','):
            if c == "from_addr":
                key = "FROM"
            elif c == "subject":
                key = "SUBJECT"
            else:
                continue
            criterions = \
                unicode(or_criterion(criterions, '%s "%s"' % (key, pattern)))
        self.mbc.criterions = [unicode("(%s)" % criterions)]

    @staticmethod
    def computequota(mbc):
        try:
            return int(float(mbc.quota_actual) \
                           / float(mbc.quota_limit) * 100)
        except (AttributeError, TypeError):
            return -1

    def getquota(self):
        return ImapListing.computequota(self.mbc)

class ImapEmail(Email):
    headernames = [
        ('From', True),
        ('To', True),
        ('Cc', True),
        ('Date', True),
        ('Subject', True),
        ]

    def __init__(self, mbox, mailid, request, dformat="DISPLAYMODE", addrfull=False, 
                 links=0):
        mformat = parameters.get_user(request.user, "DISPLAYMODE")
        self.dformat = parameters.get_user(request.user, dformat)

        self.headers = []
        self.attachments = {}
        self.imapc = get_imapconnector(request)
        msg = self.imapc.fetchmail(mbox, mailid, readonly=False,
                                   headers=self.headers_as_list)
        self.mbox = mbox
        self.mailid = mailid
        headers = msg['BODY[HEADER.FIELDS (%s)]' % self.headers_as_text]
        fallback_fmt = "html" if self.dformat == "plain" else "plain"
        
        self.bs = BodyStructure(msg['BODYSTRUCTURE'])
        data = None

        mformat = self.dformat if self.bs.contents.has_key(self.dformat) else fallback_fmt

        if len(self.bs.contents):
            bodyc = u''
            for part in self.bs.contents[mformat]:
                pnum = part['pnum']
                data = self.imapc._cmd("FETCH", mailid, "(BODY.PEEK[%s])" % pnum)
                content = decode_payload(part['encoding'],
                                         data[int(mailid)]['BODY[%s]' % pnum])
                charset = self._find_content_charset(part)
                if charset is not None:
                    try:
                        content = content.decode(charset)
                    except (UnicodeDecodeError, LookupError):
                        result = chardet.detect(content)
                        content = content.decode(result['encoding'])
                bodyc += content

            self._fetch_inlines()
            self.body = \
                getattr(self, "viewmail_%s" % mformat)(bodyc, links=links)
        else:
{'fraction_non_alphanumeric': 0.11857873712243325, 'fraction_numerical': 0.004204919756114654, 'mean_word_length': 2.213240261202432, 'pattern_counts': {'":': 11, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5755165', 'n_tokens_mistral': 4293, 'n_tokens_neox': 4132, 'n_words': 927}