<commit_before>import logging
import gc
import traceback

import anyjson
from django.conf import settings
from googleapiclient.errors import HttpError

from lily.celery import app
from .builders.label import LabelBuilder
from .builders.message import MessageBuilder
from .connector import GmailConnector, MessageNotFoundError
from .credentials import InvalidCredentialsError
from .models.models import EmailLabel, EmailMessage, NoEmailMessageId


logger = logging.getLogger(__name__)


class ManagerError(Exception):
    pass


class GmailManager(object):
    """
    Manager for handling in and output of EmailAccount that connects through Gmail Api.

    Attributes:
        connector: GmailConnector instance
        email_account: EmailAccount instance
        message_builder: MessageBuilder instance
        label_builder: LabelBuilder instance
    """
    def __init__(self, email_account):
        """
        Args:
            email_account (instance): EmailAccount instance

        Raises:
            ManagerError: if sync is not possible
        """
        self.email_account = email_account
        try:
            self.connector = GmailConnector(self.email_account)
        except InvalidCredentialsError:
            raise ManagerError
        else:
            self.message_builder = MessageBuilder(self)
            self.label_builder = LabelBuilder(self)

    def full_synchronize(self):
        message_ids = self.connector.get_all_message_id_list()

        # Check for message_ids that are saved as non email messages
        no_message_ids_in_db = set(
            NoEmailMessageId.objects.filter(
                account=self.email_account
            ).values_list('message_id', flat='true')
        )

        # Check for message_ids that are saved as email messages
        message_ids_in_db = set(
            EmailMessage.objects.filter(
                account=self.email_account
            ).values_list('message_id', flat='true')
        )

        # What do we need to do with every email message?
        for i, message_dict in enumerate(message_ids):
            logger.debug('Check for existing messages, %s/%s' % (i, len(message_ids)))
            if message_dict['id'] in no_message_ids_in_db:
                # Not an email, but chatmessage, skip
                pass
            elif message_dict['id'] not in message_ids_in_db:
                # Message is new
                app.send_task(
                    'download_email_message',
                    args=[self.email_account.id, message_dict['id']],
                    queue='email_first_sync'
                )

            else:
                # We only need to update the labels for this message
                app.send_task('update_labels_for_message', args=[self.email_account.id, message_dict['id']])

        self.connector.save_history_id()

        # Only if transaction was successful, we update the history ID
        logger.debug('Finished syncing, storing history id for %s' % self.email_account.email_address)

    def download_message(self, message_id):
        """
        Download message from Google and parse into an EmailMessage

        Arguments:
            message_id (str): message_id of the message
        """

        # If message if already downloaded, only update it
        if EmailMessage.objects.filter(account=self.email_account, message_id=message_id).exists():
            self.update_labels_for_message(message_id)
            return

        # Fetch the info from the connector and only store it if it is still out there.
        try:
            message_info = self.connector.get_message_info(message_id)
        except MessageNotFoundError:
            logger.debug('Message already deleted from remote')<commit_msg>Make sure updates of labels is also put on a low prio queue when full syncing<commit_after>import logging
import gc
import traceback

import anyjson
from django.conf import settings
from googleapiclient.errors import HttpError

from lily.celery import app
from .builders.label import LabelBuilder
from .builders.message import MessageBuilder
from .connector import GmailConnector, MessageNotFoundError
from .credentials import InvalidCredentialsError
from .models.models import EmailLabel, EmailMessage, NoEmailMessageId


logger = logging.getLogger(__name__)


class ManagerError(Exception):
    pass


class GmailManager(object):
    """
    Manager for handling in and output of EmailAccount that connects through Gmail Api.

    Attributes:
        connector: GmailConnector instance
        email_account: EmailAccount instance
        message_builder: MessageBuilder instance
        label_builder: LabelBuilder instance
    """
    def __init__(self, email_account):
        """
        Args:
            email_account (instance): EmailAccount instance

        Raises:
            ManagerError: if sync is not possible
        """
        self.email_account = email_account
        try:
            self.connector = GmailConnector(self.email_account)
        except InvalidCredentialsError:
            raise ManagerError
        else:
            self.message_builder = MessageBuilder(self)
            self.label_builder = LabelBuilder(self)

    def full_synchronize(self):
        message_ids = self.connector.get_all_message_id_list()

        # Check for message_ids that are saved as non email messages
        no_message_ids_in_db = set(
            NoEmailMessageId.objects.filter(
                account=self.email_account
            ).values_list('message_id', flat='true')
        )

        # Check for message_ids that are saved as email messages
        message_ids_in_db = set(
            EmailMessage.objects.filter(
                account=self.email_account
            ).values_list('message_id', flat='true')
        )

        # What do we need to do with every email message?
        for i, message_dict in enumerate(message_ids):
            logger.debug('Check for existing messages, %s/%s' % (i, len(message_ids)))
            if message_dict['id'] in no_message_ids_in_db:
                # Not an email, but chatmessage, skip
                pass
            elif message_dict['id'] not in message_ids_in_db:
                # Message is new
                app.send_task(
                    'download_email_message',
                    args=[self.email_account.id, message_dict['id']],
                    queue='email_first_sync'
                )

            else:
                # We only need to update the labels for this message
                app.send_task(
                    'update_labels_for_message',
                    args=[self.email_account.id, message_dict['id']],
                    queue='email_first_sync'
                )

        self.connector.save_history_id()

        # Only if transaction was successful, we update the history ID
        logger.debug('Finished syncing, storing history id for %s' % self.email_account.email_address)

    def download_message(self, message_id):
        """
        Download message from Google and parse into an EmailMessage

        Arguments:
            message_id (str): message_id of the message
        """

        # If message if already downloaded, only update it
        if EmailMessage.objects.filter(account=self.email_account, message_id=message_id).exists():
            self.update_labels_for_message(message_id)
            return

        # Fetch the info from the connector and only store it if it is still out there.
        try:
            message_info = self.connector.get_message_info(message_id)
        except MessageNotFoundError:
            logger.debug('Message already deleted from remote')
{'fraction_non_alphanumeric': 0.06631989596879063, 'fraction_numerical': 0.0, 'mean_word_length': 2.339557099435519, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2739503', 'n_tokens_mistral': 1962, 'n_tokens_neox': 1901, 'n_words': 570}