<commit_before>import time
import platform
import collections

import gevent
from gevent.lock import BoundedSemaphore
from sqlalchemy.exc import OperationalError
import psutil

from inbox.providers import providers
from inbox.config import config
from inbox.contacts.remote_sync import ContactSync
from inbox.events.remote_sync import EventSync, GoogleEventSync
from inbox.heartbeat.status import clear_heartbeat_status
from nylas.logging import get_logger
from nylas.logging.sentry import log_uncaught_errors
from inbox.models.session import session_scope
from inbox.models import Account
from inbox.scheduling.queue import QueueClient
from inbox.util.concurrency import retry_with_logging
from inbox.util.stats import statsd_client

from inbox.mailsync.backends import module_registry

USE_GOOGLE_PUSH_NOTIFICATIONS = \
    'GOOGLE_PUSH_NOTIFICATIONS' in config.get('FEATURE_FLAGS', [])

# How much time (in minutes) should all CPUs be over 90% to consider them
# overloaded.
OVERLOAD_MIN = 40
SYNC_POLL_INTERVAL = 10
NUM_CPU_SAMPLES = (OVERLOAD_MIN * 60) / SYNC_POLL_INTERVAL
NOMINAL_THRESHOLD = 90.0


class SyncService(object):
    """
    Parameters
    ----------
    process_identifier: string
        Unique identifying string for this process (currently
        <hostname>:<process_number>)
    cpu_id: int
        If a system has 4 cores, value from 0-3. (Each sync service on the
        system should get a different value.)
    poll_interval : int
        Seconds between polls for account changes.
    """
    def __init__(self, process_identifier, cpu_id,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()<commit_msg>Bring overload duration down.<commit_after>import time
import platform
import collections

import gevent
from gevent.lock import BoundedSemaphore
from sqlalchemy.exc import OperationalError
import psutil

from inbox.providers import providers
from inbox.config import config
from inbox.contacts.remote_sync import ContactSync
from inbox.events.remote_sync import EventSync, GoogleEventSync
from inbox.heartbeat.status import clear_heartbeat_status
from nylas.logging import get_logger
from nylas.logging.sentry import log_uncaught_errors
from inbox.models.session import session_scope
from inbox.models import Account
from inbox.scheduling.queue import QueueClient
from inbox.util.concurrency import retry_with_logging
from inbox.util.stats import statsd_client

from inbox.mailsync.backends import module_registry

USE_GOOGLE_PUSH_NOTIFICATIONS = \
    'GOOGLE_PUSH_NOTIFICATIONS' in config.get('FEATURE_FLAGS', [])

# How much time (in minutes) should all CPUs be over 90% to consider them
# overloaded.
OVERLOAD_MIN = 20
SYNC_POLL_INTERVAL = 10
NUM_CPU_SAMPLES = (OVERLOAD_MIN * 60) / SYNC_POLL_INTERVAL
NOMINAL_THRESHOLD = 90.0


class SyncService(object):
    """
    Parameters
    ----------
    process_identifier: string
        Unique identifying string for this process (currently
        <hostname>:<process_number>)
    cpu_id: int
        If a system has 4 cores, value from 0-3. (Each sync service on the
        system should get a different value.)
    poll_interval : int
        Seconds between polls for account changes.
    """
    def __init__(self, process_identifier, cpu_id,
                 poll_interval=SYNC_POLL_INTERVAL):
        self.host = platform.node()
        self.cpu_id = cpu_id
        self.process_identifier = process_identifier
        self.monitor_cls_for = {mod.PROVIDER: getattr(
            mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values()
{'fraction_non_alphanumeric': 0.06319407720782655, 'fraction_numerical': 0.007403490216816499, 'mean_word_length': 3.9973579920739764, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1327759', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1087, 'n_words': 328}