<commit_before>from datetime import datetime, timedelta
from requests.exceptions import HTTPError
from sqlalchemy import func

from nylas.logging import get_logger
logger = get_logger()

from inbox.basicauth import AccessNotEnabledError, OAuthError
from inbox.config import config
from inbox.sync.base_sync import BaseSyncMonitor
from inbox.models import Event, Calendar
from inbox.models.event import RecurringEvent, RecurringEventOverride
from inbox.util.debug import bind_context
from inbox.models.session import session_scope

from inbox.models.account import Account

from inbox.events.recurring import link_events
from inbox.events.google import GoogleEventsProvider


EVENT_SYNC_FOLDER_ID = -2
EVENT_SYNC_FOLDER_NAME = 'Events'
POLL_FREQUENCY = config.get('CALENDAR_POLL_FREQUENCY', 300)

MAX_TIME_WITHOUT_SYNC = timedelta(seconds=3600)


class EventSync(BaseSyncMonitor):
    """Per-account event sync engine."""

    def __init__(self, email_address, provider_name, account_id, namespace_id,
                 poll_frequency=POLL_FREQUENCY):
        bind_context(self, 'eventsync', account_id)
        # Only Google for now, can easily parametrize by provider later.
        self.provider = GoogleEventsProvider(account_id, namespace_id)

        BaseSyncMonitor.__init__(self,
                                 account_id,
                                 namespace_id,
                                 email_address,
                                 EVENT_SYNC_FOLDER_ID,
                                 EVENT_SYNC_FOLDER_NAME,
                                 provider_name,
                                 poll_frequency=poll_frequency,
                                 scope='calendar')

    def sync(self):
        """Query a remote provider for updates and persist them to the
        database. This function runs every `self.poll_frequency`.
        """
        self.log.info('syncing events')

        try:
            deleted_uids, calendar_changes = self.provider.sync_calendars()
        except AccessNotEnabledError:
            self.log.warning(
                'Access to provider calendar API not enabled; bypassing sync')
            return
        with session_scope(self.namespace_id) as db_session:
            handle_calendar_deletes(self.namespace_id, deleted_uids,
                                    self.log, db_session)
            calendar_uids_and_ids = handle_calendar_updates(self.namespace_id,
                                                            calendar_changes,
                                                            self.log,
                                                            db_session)
            db_session.commit()

        for (uid, id_) in calendar_uids_and_ids:
            # Get a timestamp before polling, so that we don't subsequently
            # miss remote updates that happen while the poll loop is executing.
            sync_timestamp = datetime.utcnow()
            with session_scope(self.namespace_id) as db_session:
                last_sync = db_session.query(Calendar.last_synced).filter(
                    Calendar.id == id_).scalar()

            event_changes = self.provider.sync_events(
                uid, sync_from_time=last_sync)

            with session_scope(self.namespace_id) as db_session:
                handle_event_updates(self.namespace_id, id_, event_changes,
                                     self.log, db_session)
                cal = db_session.query(Calendar).get(id_)
                cal.last_synced = sync_timestamp
                db_session.commit()


def handle_calendar_deletes(namespace_id, deleted_calendar_uids, log,
                            db_session):
    """Delete any local Calendar rows with uid in `deleted_calendar_uids`. This
    delete cascades to associated events (if the calendar is gone, so are all
    of its events)."""
    deleted_count = 0
    for uid in deleted_calendar_uids:
        local_calendar = db_session.query(Calendar).filter(
            Calendar.namespace_id == namespace_id,
            Calendar.uid == uid).first()
        if local_calendar is not None:
            # Cascades to associated events via SQLAlchemy 'delete' cascade
            db_session.delete(local_calendar)
            deleted_count += 1
    log.info('deleted calendars', deleted=deleted_count)


def handle_calendar_updates(namespace_id, calendars, log, db_session):
    """Persists new or updated Calendar objects to the database."""
    ids_ = []
    added_count = 0
    updated_count = 0
    for calendar in calendars:
        assert calendar.uid is not None, 'Got remote item with null uid'

        local_calendar = db_session.query(Calendar).filter(
            Calendar.namespace_id == namespace_id,
            Calendar.uid == calendar.uid).first()

        if local_calendar is not None:
            local_calendar.update(calendar)
            updated_count += 1
        else:
            local_calendar = Calendar(namespace_id=namespace_id)
            local_calendar.update(calendar)
            db_session.add(local_calendar)
            db_session.flush()
            added_count += 1

        ids_.append((local_calendar.uid, local_calendar.id))

    log.info('synced added and updated calendars', added=added_count,
             updated=updated_count)
    return ids_


def handle_event_updates(namespace_id, calendar_id, events, log, db_session):
    """Persists new or updated Event objects to the database."""
    added_count = 0
    updated_count = 0
    existing_event_count = db_session.query(func.count(Event.id)).filter(
        Event.namespace_id == namespace_id,
        Event.calendar_id == calendar_id).scalar()
    for event in events:
        assert event.uid is not None, 'Got remote item with null uid'

        local_event = None
        if existing_event_count:
            # Skip this lookup if there are no local events at all, for faster
            # first sync.
            local_event = db_session.query(Event).filter(
                Event.namespace_id == namespace_id,
                Event.calendar_id == calendar_id,
                Event.uid == event.uid).first()

        if local_event is not None:
            # We also need to mark all overrides as cancelled if we're
            # cancelling a recurring event. However, note the original event
            # may not itself be recurring (recurrence may have been added).
            if isinstance(local_event, RecurringEvent) and \
                    event.status == 'cancelled' and \
                    local_event.status != 'cancelled':
                for override in local_event.overrides:
                    override.status = 'cancelled'
<commit_msg>Replace potentially expensive COUNT with EXISTS.<commit_after>from datetime import datetime, timedelta
from requests.exceptions import HTTPError

from nylas.logging import get_logger
logger = get_logger()

from inbox.basicauth import AccessNotEnabledError, OAuthError
from inbox.config import config
from inbox.sync.base_sync import BaseSyncMonitor
from inbox.models import Event, Calendar
from inbox.models.event import RecurringEvent, RecurringEventOverride
from inbox.util.debug import bind_context
from inbox.models.session import session_scope

from inbox.models.account import Account

from inbox.events.recurring import link_events
from inbox.events.google import GoogleEventsProvider


EVENT_SYNC_FOLDER_ID = -2
EVENT_SYNC_FOLDER_NAME = 'Events'
POLL_FREQUENCY = config.get('CALENDAR_POLL_FREQUENCY', 300)

MAX_TIME_WITHOUT_SYNC = timedelta(seconds=3600)


class EventSync(BaseSyncMonitor):
    """Per-account event sync engine."""

    def __init__(self, email_address, provider_name, account_id, namespace_id,
                 poll_frequency=POLL_FREQUENCY):
        bind_context(self, 'eventsync', account_id)
        # Only Google for now, can easily parametrize by provider later.
        self.provider = GoogleEventsProvider(account_id, namespace_id)

        BaseSyncMonitor.__init__(self,
                                 account_id,
                                 namespace_id,
                                 email_address,
                                 EVENT_SYNC_FOLDER_ID,
                                 EVENT_SYNC_FOLDER_NAME,
                                 provider_name,
                                 poll_frequency=poll_frequency,
                                 scope='calendar')

    def sync(self):
        """Query a remote provider for updates and persist them to the
        database. This function runs every `self.poll_frequency`.
        """
        self.log.info('syncing events')

        try:
            deleted_uids, calendar_changes = self.provider.sync_calendars()
        except AccessNotEnabledError:
            self.log.warning(
                'Access to provider calendar API not enabled; bypassing sync')
            return
        with session_scope(self.namespace_id) as db_session:
            handle_calendar_deletes(self.namespace_id, deleted_uids,
                                    self.log, db_session)
            calendar_uids_and_ids = handle_calendar_updates(self.namespace_id,
                                                            calendar_changes,
                                                            self.log,
                                                            db_session)
            db_session.commit()

        for (uid, id_) in calendar_uids_and_ids:
            # Get a timestamp before polling, so that we don't subsequently
            # miss remote updates that happen while the poll loop is executing.
            sync_timestamp = datetime.utcnow()
            with session_scope(self.namespace_id) as db_session:
                last_sync = db_session.query(Calendar.last_synced).filter(
                    Calendar.id == id_).scalar()

            event_changes = self.provider.sync_events(
                uid, sync_from_time=last_sync)

            with session_scope(self.namespace_id) as db_session:
                handle_event_updates(self.namespace_id, id_, event_changes,
                                     self.log, db_session)
                cal = db_session.query(Calendar).get(id_)
                cal.last_synced = sync_timestamp
                db_session.commit()


def handle_calendar_deletes(namespace_id, deleted_calendar_uids, log,
                            db_session):
    """Delete any local Calendar rows with uid in `deleted_calendar_uids`. This
    delete cascades to associated events (if the calendar is gone, so are all
    of its events)."""
    deleted_count = 0
    for uid in deleted_calendar_uids:
        local_calendar = db_session.query(Calendar).filter(
            Calendar.namespace_id == namespace_id,
            Calendar.uid == uid).first()
        if local_calendar is not None:
            # Cascades to associated events via SQLAlchemy 'delete' cascade
            db_session.delete(local_calendar)
            deleted_count += 1
    log.info('deleted calendars', deleted=deleted_count)


def handle_calendar_updates(namespace_id, calendars, log, db_session):
    """Persists new or updated Calendar objects to the database."""
    ids_ = []
    added_count = 0
    updated_count = 0
    for calendar in calendars:
        assert calendar.uid is not None, 'Got remote item with null uid'

        local_calendar = db_session.query(Calendar).filter(
            Calendar.namespace_id == namespace_id,
            Calendar.uid == calendar.uid).first()

        if local_calendar is not None:
            local_calendar.update(calendar)
            updated_count += 1
        else:
            local_calendar = Calendar(namespace_id=namespace_id)
            local_calendar.update(calendar)
            db_session.add(local_calendar)
            db_session.flush()
            added_count += 1

        ids_.append((local_calendar.uid, local_calendar.id))

    log.info('synced added and updated calendars', added=added_count,
             updated=updated_count)
    return ids_


def handle_event_updates(namespace_id, calendar_id, events, log, db_session):
    """Persists new or updated Event objects to the database."""
    added_count = 0
    updated_count = 0
    existing_event_query = db_session.query(Event).filter(
        Event.namespace_id == namespace_id,
        Event.calendar_id == calendar_id).exists()
    events_exist = db_session.query(existing_event_query).scalar()
    for event in events:
        assert event.uid is not None, 'Got remote item with null uid'

        local_event = None
        if events_exist:
            # Skip this lookup if there are no local events at all, for faster
            # first sync.
            local_event = db_session.query(Event).filter(
                Event.namespace_id == namespace_id,
                Event.calendar_id == calendar_id,
                Event.uid == event.uid).first()

        if local_event is not None:
            # We also need to mark all overrides as cancelled if we're
            # cancelling a recurring event. However, note the original event
            # may not itself be recurring (recurrence may have been added).
            if isinstance(local_event, RecurringEvent) and \
                    event.status == 'cancelled' and \
                    local_event.status != 'cancelled':
                for override in local_event.overrides:
                    override.status = 'cancelled'

{'fraction_non_alphanumeric': 0.06713807219947897, 'fraction_numerical': 0.002381838481577968, 'mean_word_length': 2.100138440239963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7457736', 'n_tokens_mistral': 3527, 'n_tokens_neox': 3377, 'n_words': 925}