<commit_before>""" Helper functions for actions that operate on accounts.

These could be methods of ImapAccount, but separating them gives us more
flexibility with calling code, as most don't need any attributes of the account
object other than the ID, to limit the action.

Types returned for data are the column types defined via SQLAlchemy.

Eventually we're going to want a better way of ACLing functions that operate on
accounts.
"""
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound

from inbox.contacts.process_mail import update_contacts_from_message
from inbox.models.message import Message
from inbox.models.folder import Folder
from inbox.models.backends.imap import ImapUid, ImapFolderInfo
from inbox.models.util import reconcile_message

from inbox.log import get_logger
log = get_logger()


def all_uids(account_id, session, folder_name):
    return {uid for uid, in session.query(ImapUid.msg_uid).join(Folder).filter(
        ImapUid.account_id == account_id,
        Folder.name == folder_name)}


def update_thread_labels(thread, folder_name, g_labels, db_session):
    """ Make sure `thread` has all the right labels. """
    existing_labels = {folder.name.lower() for folder in thread.folders
                       if folder.name is not None} | \
                      {folder.canonical_name for folder in thread.folders
                       if folder.canonical_name is not None}

    new_labels = {l.lstrip('\\').lower() if isinstance(l, unicode)
                  else unicode(l) for l in g_labels if l is not None}
    new_labels.add(folder_name.lower())

    # Remove labels that have been deleted -- note that the \Inbox, \Sent,
    # \Important, \Starred, and \Drafts labels are per-message, not per-thread,
    # but since we always work at the thread level, _we_ apply the label to the
    # whole thread.
    # TODO: properly aggregate \Inbox, \Sent, \Important, and \Drafts
    # per-message so we can detect deletions properly.
    folders_to_discard = []
    for folder in thread.folders:
        if folder.canonical_name not in ('inbox', 'sent', 'drafts',
                                         'important', 'starred', 'all'):
            if folder.lowercase_name not in new_labels:
                folders_to_discard.append(folder)
    for folder in folders_to_discard:
        thread.folders.discard(folder)

    # add new labels
    for label in new_labels:
        if label.lower() not in existing_labels:
            # The problem here is that Gmail's attempt to squash labels and
            # IMAP folders into the same abstraction doesn't work perfectly. In
            # particular, there is a '[Gmail]/Sent' folder, but *also* a 'Sent'
            # label, and so on. We handle this by only maintaining one folder
            # object that encapsulates both of these. If a Gmail user does not
            # have these folders enabled via IMAP, we create Folder rows
            # with no 'name' attribute and fill in the 'name' if the account
            # is later reconfigured.
            canonical_labels = {
                'sent': thread.namespace.account.sent_folder,
                'draft': thread.namespace.account.drafts_folder,
                'starred': thread.namespace.account.starred_folder,
                'important': thread.namespace.account.important_folder}
            if label in canonical_labels:
                folder = canonical_labels[label]
                if folder:
                    thread.folders.add(folder)
                else:
                    folder = Folder.find_or_create(
                        db_session, thread.namespace.account, None, label)
                    thread.folders.add(folder)
            else:
                folder = Folder.find_or_create(db_session,
                                               thread.namespace.account, label)
                thread.folders.add(folder)
    return new_labels


def update_metadata(account_id, session, folder_name, uids, new_flags):
    """
    Update flags (the only metadata that can change).

    Make sure you're holding a db write lock on the account. (We don't try
    to grab the lock in here in case the caller needs to put higher-level
    functionality in the lock.)

    """
    if uids:
        # When we're updating multiple messages in the same thread and a
        # call to update_thread_labels() results in a Folder.find_or_create(),
        # a subsequent access of that thread would trigger a flush,
        # raising an IntegrityError - so disable autoflush.
        with session.no_autoflush:
            for item in session.query(ImapUid).join(Folder)\
                    .filter(ImapUid.account_id == account_id,
                            ImapUid.msg_uid.in_(uids),
                            Folder.name == folder_name)\
                    .options(joinedload(ImapUid.message)):
                flags = new_flags[item.msg_uid].flags
                thread = item.message.thread
                if hasattr(new_flags[item.msg_uid], 'labels'):
                    labels = new_flags[item.msg_uid].labels
                    update_thread_labels(thread, folder_name, labels, session)
                else:
                    labels = None
                item.update_imap_flags(flags, labels)

                unread_status_changed = item.message.is_read != item.is_seen

                item.message.is_draft = item.is_draft
                item.message.is_read = item.is_seen

                # We have to update the thread's read tag separately.
                if unread_status_changed:
                    unread_tag = thread.namespace.tags['unread']
                    if not item.message.is_read:
                        thread.apply_tag(unread_tag)
                    elif all(m.is_read for m in thread.messages):
                        thread.remove_tag(unread_tag)


def remove_messages(account_id, session, uids, folder):
    """ Make sure you're holding a db write lock on the account. (We don't try
        to grab the lock in here in case the caller needs to put higher-level
        functionality in the lock.)
    """
    if uids:
        deletes = session.query(ImapUid).join(Folder).filter(
            ImapUid.account_id == account_id,
            Folder.name == folder,
            ImapUid.msg_uid.in_(uids)).all()

        # Remove message if he has no associated UIDs.
        # This may cause the message to disappear transiently
        # when moved between folders. The delay should be short
        # enough to be unnoticeable.
        #
        # FIXME @karim: we shouldn't be deleting messages --
        # the clean way to handle this problem is have a daemon
        # detect "dangling" threads and messages and collect
        # them periodically (tracked in T477)
        affected_messages = {uid.message for uid in deletes}

        for uid in deletes:
            session.delete(uid)
        session.commit()

        messages_to_delete = {m for m in affected_messages if not m.imapuids}

        # Because we need to update thread folders and tags, threads are
        # 'affected' even if we're not removing messages from them.
        affected_threads = {m.thread for m in affected_messages}

        for message in messages_to_delete:
            session.delete(message)
        session.commit()

        for thread in affected_threads:
            if not thread.messages:
                session.delete(thread)
            else:
                # We've deleted messages, we've got to update thread.folders.
                # We use a (temporary) heuristic for this: a thread has
                # the same folders as its latest message.

                # NB: thread.messages is sorted by date ascending.
                most_recent_message = thread.messages[-1]
                mrm_folders = [uid.folder
                               for uid in most_recent_message.imapuids]<commit_msg>Fix NoneType error in remove_messages().<commit_after>""" Helper functions for actions that operate on accounts.

These could be methods of ImapAccount, but separating them gives us more
flexibility with calling code, as most don't need any attributes of the account
object other than the ID, to limit the action.

Types returned for data are the column types defined via SQLAlchemy.

Eventually we're going to want a better way of ACLing functions that operate on
accounts.
"""
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound

from inbox.contacts.process_mail import update_contacts_from_message
from inbox.models.message import Message
from inbox.models.folder import Folder
from inbox.models.backends.imap import ImapUid, ImapFolderInfo
from inbox.models.util import reconcile_message

from inbox.log import get_logger
log = get_logger()


def all_uids(account_id, session, folder_name):
    return {uid for uid, in session.query(ImapUid.msg_uid).join(Folder).filter(
        ImapUid.account_id == account_id,
        Folder.name == folder_name)}


def update_thread_labels(thread, folder_name, g_labels, db_session):
    """ Make sure `thread` has all the right labels. """
    existing_labels = {folder.name.lower() for folder in thread.folders
                       if folder.name is not None} | \
                      {folder.canonical_name for folder in thread.folders
                       if folder.canonical_name is not None}

    new_labels = {l.lstrip('\\').lower() if isinstance(l, unicode)
                  else unicode(l) for l in g_labels if l is not None}
    new_labels.add(folder_name.lower())

    # Remove labels that have been deleted -- note that the \Inbox, \Sent,
    # \Important, \Starred, and \Drafts labels are per-message, not per-thread,
    # but since we always work at the thread level, _we_ apply the label to the
    # whole thread.
    # TODO: properly aggregate \Inbox, \Sent, \Important, and \Drafts
    # per-message so we can detect deletions properly.
    folders_to_discard = []
    for folder in thread.folders:
        if folder.canonical_name not in ('inbox', 'sent', 'drafts',
                                         'important', 'starred', 'all'):
            if folder.lowercase_name not in new_labels:
                folders_to_discard.append(folder)
    for folder in folders_to_discard:
        thread.folders.discard(folder)

    # add new labels
    for label in new_labels:
        if label.lower() not in existing_labels:
            # The problem here is that Gmail's attempt to squash labels and
            # IMAP folders into the same abstraction doesn't work perfectly. In
            # particular, there is a '[Gmail]/Sent' folder, but *also* a 'Sent'
            # label, and so on. We handle this by only maintaining one folder
            # object that encapsulates both of these. If a Gmail user does not
            # have these folders enabled via IMAP, we create Folder rows
            # with no 'name' attribute and fill in the 'name' if the account
            # is later reconfigured.
            canonical_labels = {
                'sent': thread.namespace.account.sent_folder,
                'draft': thread.namespace.account.drafts_folder,
                'starred': thread.namespace.account.starred_folder,
                'important': thread.namespace.account.important_folder}
            if label in canonical_labels:
                folder = canonical_labels[label]
                if folder:
                    thread.folders.add(folder)
                else:
                    folder = Folder.find_or_create(
                        db_session, thread.namespace.account, None, label)
                    thread.folders.add(folder)
            else:
                folder = Folder.find_or_create(db_session,
                                               thread.namespace.account, label)
                thread.folders.add(folder)
    return new_labels


def update_metadata(account_id, session, folder_name, uids, new_flags):
    """
    Update flags (the only metadata that can change).

    Make sure you're holding a db write lock on the account. (We don't try
    to grab the lock in here in case the caller needs to put higher-level
    functionality in the lock.)

    """
    if uids:
        # When we're updating multiple messages in the same thread and a
        # call to update_thread_labels() results in a Folder.find_or_create(),
        # a subsequent access of that thread would trigger a flush,
        # raising an IntegrityError - so disable autoflush.
        with session.no_autoflush:
            for item in session.query(ImapUid).join(Folder)\
                    .filter(ImapUid.account_id == account_id,
                            ImapUid.msg_uid.in_(uids),
                            Folder.name == folder_name)\
                    .options(joinedload(ImapUid.message)):
                flags = new_flags[item.msg_uid].flags
                thread = item.message.thread
                if hasattr(new_flags[item.msg_uid], 'labels'):
                    labels = new_flags[item.msg_uid].labels
                    update_thread_labels(thread, folder_name, labels, session)
                else:
                    labels = None
                item.update_imap_flags(flags, labels)

                unread_status_changed = item.message.is_read != item.is_seen

                item.message.is_draft = item.is_draft
                item.message.is_read = item.is_seen

                # We have to update the thread's read tag separately.
                if unread_status_changed:
                    unread_tag = thread.namespace.tags['unread']
                    if not item.message.is_read:
                        thread.apply_tag(unread_tag)
                    elif all(m.is_read for m in thread.messages):
                        thread.remove_tag(unread_tag)


def remove_messages(account_id, session, uids, folder):
    """ Make sure you're holding a db write lock on the account. (We don't try
        to grab the lock in here in case the caller needs to put higher-level
        functionality in the lock.)
    """
    if uids:
        deletes = session.query(ImapUid).join(Folder).filter(
            ImapUid.account_id == account_id,
            Folder.name == folder,
            ImapUid.msg_uid.in_(uids)).all()

        # Remove message if he has no associated UIDs.
        # This may cause the message to disappear transiently
        # when moved between folders. The delay should be short
        # enough to be unnoticeable.
        #
        # FIXME @karim: we shouldn't be deleting messages --
        # the clean way to handle this problem is have a daemon
        # detect "dangling" threads and messages and collect
        # them periodically (tracked in T477)
        affected_messages = {uid.message for uid in deletes
                             if uid.message is not None}

        for uid in deletes:
            session.delete(uid)
        session.commit()

        messages_to_delete = {m for m in affected_messages if not m.imapuids}

        # Because we need to update thread folders and tags, threads are
        # 'affected' even if we're not removing messages from them.
        affected_threads = {m.thread for m in affected_messages}

        for message in messages_to_delete:
            session.delete(message)
        session.commit()

        for thread in affected_threads:
            if not thread.messages:
                session.delete(thread)
            else:
                # We've deleted messages, we've got to update thread.folders.
                # We use a (temporary) heuristic for this: a thread has
                # the same folders as its latest message.

                # NB: thread.messages is sorted by date ascending.
                most_recent_message = thread.messages[-1]
                mrm_folders = [uid.folder
                               for uid in most_recent_message.imapuids]
{'fraction_non_alphanumeric': 0.0707798308800501, 'fraction_numerical': 0.0005010961478233636, 'mean_word_length': 2.059206744587086, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7574084', 'n_tokens_mistral': 4312, 'n_tokens_neox': 4065, 'n_words': 1454}