<commit_before>import os
import pygmail.errors
from .fs import sanatize_filename, unique_filename
from pygmail.account import Account

ATTACHMENT_MIMES = ('image/jpeg', 'image/png', 'image/gif')

class GmailImageExtractor(object):
    """Image extrating class which handles connecting to gmail on behalf of
    a user over IMAP, extracts images from messages in a Gmail account,
    writes them to disk, allows users to delete extracted images, and then
    syncronizes the messages in the gmail account by deleting any images
    deleted from the file system from their corresponding message in the
    account.
    """

    def __init__(self, dest, email, password, limit=None, batch=10, replace=False):
        """
        Args:
            dest     -- the path on the file system where images should be
                        extracted and written to.
            email -- the username of the Gmail account to connect to
            password -- the password of the Gmail account to connect to

        Keyword Args:
            limit   -- an optional limit of the total number of messages to
                       download from the gmail account.
            batch   -- the maximum number of messages to download from Gmail
                       at the same time.
            replace -- whether to rewrite the messages in the Gmail account
                       in place (True) or to just write a second, parallel
                       copy of the altered message and leave the original
                       version alone.

        raise:
            ValueError -- If the given dest path to write extracted images to
                          is not writeable.
        """
        self.dest = dest

        if not self.validate_path():
            raise ValueError("{0} is not a writeable directory".format(dest))

        self.limit = limit
        self.batch = batch
        self.replace = replace
        self.email = email
        self.password = password

    def validate_path(self):
        """Checks to see the currently selected destiation path, for where
        extracted images should be written, is a valid path that we can
        read and write from.

        Return:
            A boolean description of whether the currently selected destination
            is a valid path we can read from and write to.
        """
        if not os.path.isdir(self.dest):
            return False
        elif not os.access(self.dest, os.W_OK):
            return False
        else:
            return True

    def connect(self):
        """Attempts to connect to Gmail using the username and password provided
        at instantiation.

        Returns:
            Returns a boolean description of whether we were able to connect
            to Gmail using the current parameters.
        """
        mail = Account(self.email, password=self.password)
        trash_folder = mail.trash_mailbox()
        if pygmail.errors.is_error(trash_folder):
            return False
        else:
            self.mail = mail
            self.trash_folder = trash_folder
            self.inbox = mail.all_mailbox()
            return True

    def num_messages_with_attachments(self):
        """Checks to see how many Gmail messages have attachments in the
        currently connected gmail account.

        This should only be called after having succesfully connected to Gmail.

        Return:
            The number of messages in the Gmail account that have at least one
            attachment (as advertised by Gmail).
        """
        limit = self.limit if self.limit > 0 else False
        gm_ids = self.inbox.search("has:attachment", gm_ids=True, limit=limit)
        return len(gm_ids)

    def extract(self, callback=None):
        """Extracts images from Gmail messages and writes them to the
        path set at instantiation.

        Keyword Args:
            callback -- An optional funciton that will be called with updates
                        about the image extraction process. If provided,
                        will be called with either the following arguments

                        ('image', attachment_name, disk_name)
                        when writing a message to disk, where
                        `attachment_name` is the name of the attachment
                        as advertised in the Email message, and `disk_name`
                        is the name of the file as written to disk.

                        ('message', first)
                        when fetching messages from Gmail, where `first` is the
                        index of the current message being downloaded.

        Returns:
            The number of images written to disk.
        """
        def _cb(*args):
            if callback:
                callback(*args)

        attachment_count = 0
        num_messages = 0
        offset = 0
        per_page = min(self.batch, self.limit) if self.limit else self.batch
        # Keep track of which attachments belong to which messages.  Do this
        # by keeping track of all attachments downloaded to the filesystem
        # (used as the dict key) and pairing it with two values, the gmail
        # message id and the hash of the attachment (so that we can uniquely
        # identify the attachment again)
        self.mapping = {}
        hit_limit = False
        while True and not hit_limit:
            _cb('message', offset + 1)
            messages = self.inbox.search("has:attachment", full=True,
                                         limit=per_page, offset=offset)
            if len(messages) == 0:
                break
            for msg in messages:
                for att in msg.attachments():
                    if att.type in ATTACHMENT_MIMES:
                        poss_fname = "{0} - {1}".format(msg.subject, att.name())
                        safe_fname = sanatize_filename(poss_fname)<commit_msg>small bug fix for unicode filenames<commit_after>import os
import pygmail.errors
from .fs import sanatize_filename, unique_filename
from pygmail.account import Account

ATTACHMENT_MIMES = ('image/jpeg', 'image/png', 'image/gif')

class GmailImageExtractor(object):
    """Image extrating class which handles connecting to gmail on behalf of
    a user over IMAP, extracts images from messages in a Gmail account,
    writes them to disk, allows users to delete extracted images, and then
    syncronizes the messages in the gmail account by deleting any images
    deleted from the file system from their corresponding message in the
    account.
    """

    def __init__(self, dest, email, password, limit=None, batch=10, replace=False):
        """
        Args:
            dest     -- the path on the file system where images should be
                        extracted and written to.
            email -- the username of the Gmail account to connect to
            password -- the password of the Gmail account to connect to

        Keyword Args:
            limit   -- an optional limit of the total number of messages to
                       download from the gmail account.
            batch   -- the maximum number of messages to download from Gmail
                       at the same time.
            replace -- whether to rewrite the messages in the Gmail account
                       in place (True) or to just write a second, parallel
                       copy of the altered message and leave the original
                       version alone.

        raise:
            ValueError -- If the given dest path to write extracted images to
                          is not writeable.
        """
        self.dest = dest

        if not self.validate_path():
            raise ValueError("{0} is not a writeable directory".format(dest))

        self.limit = limit
        self.batch = batch
        self.replace = replace
        self.email = email
        self.password = password

    def validate_path(self):
        """Checks to see the currently selected destiation path, for where
        extracted images should be written, is a valid path that we can
        read and write from.

        Return:
            A boolean description of whether the currently selected destination
            is a valid path we can read from and write to.
        """
        if not os.path.isdir(self.dest):
            return False
        elif not os.access(self.dest, os.W_OK):
            return False
        else:
            return True

    def connect(self):
        """Attempts to connect to Gmail using the username and password provided
        at instantiation.

        Returns:
            Returns a boolean description of whether we were able to connect
            to Gmail using the current parameters.
        """
        mail = Account(self.email, password=self.password)
        trash_folder = mail.trash_mailbox()
        if pygmail.errors.is_error(trash_folder):
            return False
        else:
            self.mail = mail
            self.trash_folder = trash_folder
            self.inbox = mail.all_mailbox()
            return True

    def num_messages_with_attachments(self):
        """Checks to see how many Gmail messages have attachments in the
        currently connected gmail account.

        This should only be called after having succesfully connected to Gmail.

        Return:
            The number of messages in the Gmail account that have at least one
            attachment (as advertised by Gmail).
        """
        limit = self.limit if self.limit > 0 else False
        gm_ids = self.inbox.search("has:attachment", gm_ids=True, limit=limit)
        return len(gm_ids)

    def extract(self, callback=None):
        """Extracts images from Gmail messages and writes them to the
        path set at instantiation.

        Keyword Args:
            callback -- An optional funciton that will be called with updates
                        about the image extraction process. If provided,
                        will be called with either the following arguments

                        ('image', attachment_name, disk_name)
                        when writing a message to disk, where
                        `attachment_name` is the name of the attachment
                        as advertised in the Email message, and `disk_name`
                        is the name of the file as written to disk.

                        ('message', first)
                        when fetching messages from Gmail, where `first` is the
                        index of the current message being downloaded.

        Returns:
            The number of images written to disk.
        """
        def _cb(*args):
            if callback:
                callback(*args)

        attachment_count = 0
        num_messages = 0
        offset = 0
        per_page = min(self.batch, self.limit) if self.limit else self.batch
        # Keep track of which attachments belong to which messages.  Do this
        # by keeping track of all attachments downloaded to the filesystem
        # (used as the dict key) and pairing it with two values, the gmail
        # message id and the hash of the attachment (so that we can uniquely
        # identify the attachment again)
        self.mapping = {}
        hit_limit = False
        while True and not hit_limit:
            _cb('message', offset + 1)
            messages = self.inbox.search("has:attachment", full=True,
                                         limit=per_page, offset=offset)
            if len(messages) == 0:
                break
            for msg in messages:
                for att in msg.attachments():
                    if att.type in ATTACHMENT_MIMES:
                        poss_fname = u"{0} - {1}".format(msg.subject, att.name())
                        safe_fname = sanatize_filename(poss_fname)
{'fraction_non_alphanumeric': 0.055710306406685235, 'fraction_numerical': 0.0018570102135561746, 'mean_word_length': 1.746407046824293, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4706871', 'n_tokens_mistral': 2819, 'n_tokens_neox': 2704, 'n_words': 1218}