<commit_before>import datetime
import base64
import email
import imaplib
import StringIO

from django.conf import settings
from django.contrib.auth.models import User

from mooch.logging.models import LogEntry, LogEntryFile


class IMAPReader(object):
    def __init__(self):
        self.client = imaplib.IMAP4_SSL(settings.IMAP_HOST)
        self.client.login(settings.IMAP_USERNAME, settings.IMAP_PASSWORD)
        self.client.select('INBOX')

    def unread_messages(self):
        return self.client.search(None, 'UNSEEN')[1][0].split()

    def mark_seen(self, message):
        return self.client.store(message, '+FLAGS', r'\Seen')

    def fetch(self, message):
        return email.message_from_string(self.client.fetch(message, '(RFC822)')[1][0][1])

    def fetch_and_save_unseen(self):
        def handler(entry, files, msg):
            """
            Recursively parse multipart mails
            """

            if msg.is_multipart():
                for part in msg.get_payload():
                    handler(entry, files, part)
                return

            if msg.get_content_maintype() == 'text':
                entry.message += msg.get_payload()
            elif msg.get_content_maintype() == 'image':
                files.append((
                    msg.get_filename(), # filename
                    msg.get_filename(), # title
                    msg.get_content_type(),
                    base64.decodestring(msg.get_payload()),
                    ))
            else:
                print 'Cannot handle type %s yet' % msg.get_content_type()

        entries = []
        for msgid in self.unread_messages():
            entry = LogEntry(account=User.objects.all()[0], source='EML')
            files = []

            msg = self.fetch(msgid)
            handler(entry, files, msg)

            for key, value in msg.items():
                if key == 'Subject':
                    entry.title = value
                elif key == 'Date':
                    entry.reported = datetime.datetime(*email.utils.parsedate(value)[:7])
                elif key == 'From':
                    entry.source_detail = value

            if not entry.title:
                entry.title = entry.message[:100]

            entry.save()

            from django.core.files.uploadedfile import InMemoryUploadedFile

            for filename, title, contenttype, data in files:
                f = LogEntryFile(
                    logentry=entry,
                    title=title,
                    )
                f_obj = InMemoryUploadedFile(
                    StringIO.StringIO(data),
                    'file',
                    filename,
                    contenttype,
                    len(data),
                    'utf-8',
                    )
                f.file.save(filename, f_obj)
                f.save()

            #self.mark_seen(msgid)
            entries.append(entry)

        return entries
<commit_msg>Mark added messages as seen<commit_after>import datetime
import base64
import email
import imaplib
import StringIO

from django.conf import settings
from django.contrib.auth.models import User

from mooch.logging.models import LogEntry, LogEntryFile


class IMAPReader(object):
    def __init__(self):
        self.client = imaplib.IMAP4_SSL(settings.IMAP_HOST)
        self.client.login(settings.IMAP_USERNAME, settings.IMAP_PASSWORD)
        self.client.select('INBOX')

    def unread_messages(self):
        return self.client.search(None, 'UNSEEN')[1][0].split()

    def mark_seen(self, message):
        return self.client.store(message, '+FLAGS', r'\Seen')

    def fetch(self, message):
        return email.message_from_string(self.client.fetch(message, '(RFC822)')[1][0][1])

    def fetch_and_save_unseen(self):
        def handler(entry, files, msg):
            """
            Recursively parse multipart mails
            """

            if msg.is_multipart():
                for part in msg.get_payload():
                    handler(entry, files, part)
                return

            if msg.get_content_maintype() == 'text':
                entry.message += msg.get_payload()
            elif msg.get_content_maintype() == 'image':
                files.append((
                    msg.get_filename(), # filename
                    msg.get_filename(), # title
                    msg.get_content_type(),
                    base64.decodestring(msg.get_payload()),
                    ))
            else:
                print 'Cannot handle type %s yet' % msg.get_content_type()

        entries = []
        for msgid in self.unread_messages():
            entry = LogEntry(account=User.objects.all()[0], source='EML')
            files = []

            msg = self.fetch(msgid)
            handler(entry, files, msg)

            for key, value in msg.items():
                if key == 'Subject':
                    entry.title = value
                elif key == 'Date':
                    entry.reported = datetime.datetime(*email.utils.parsedate(value)[:7])
                elif key == 'From':
                    entry.source_detail = value

            if not entry.title:
                entry.title = entry.message[:100]

            entry.save()

            from django.core.files.uploadedfile import InMemoryUploadedFile

            for filename, title, contenttype, data in files:
                f = LogEntryFile(
                    logentry=entry,
                    title=title,
                    )
                f_obj = InMemoryUploadedFile(
                    StringIO.StringIO(data),
                    'file',
                    filename,
                    contenttype,
                    len(data),
                    'utf-8',
                    )
                f.file.save(filename, f_obj)
                f.save()

            self.mark_seen(msgid)
            entries.append(entry)

        return entries

{'fraction_non_alphanumeric': 0.10079084637388525, 'fraction_numerical': 0.0063940770654551575, 'mean_word_length': 1.523991507430998, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3669086', 'n_tokens_mistral': 1630, 'n_tokens_neox': 1521, 'n_words': 332}