<commit_before>import os
import time
import re
import logging
import random
import gc
import asyncio
from datetime import datetime, timedelta

from . import web, searchers, retrievers, notifiers, providers
from .toolbox import FutureProgress, singleton
from .toolbox.utils import fsname
from .toolbox.dateutils import to_timestamp
from .toolbox.config import get_default
from .tvdb import TVDB
from .config import config
from .utils import fixsep
from .searchers import SearcherError
from .retrievers import RetrieverError, RetrieverSoftError, RetrieverHardError, RetrieverAborted, RetrieverAbortedHard, RetrieverAbortedSoft
from .notifiers import NotifierError

log = logging.getLogger('stagehand.manager')


class Manager:
    def __init__(self, paths, *, loop=None):
        super().__init__()
        self.paths = paths
        self.loop = loop or asyncio.get_event_loop()
        self._retrieve_queue_event = asyncio.Event()
        # A list of episodes to be retrieved.
        # [(Episode, [SearchResult, ...]), (Episode, [SearchResult, ...])]
        self._retrieve_queue = []
        # Like _retrieve_queue, but these are the episodes actively being processed.
        # The first SearchResult is the one being retrieved.
        self._retrieve_queue_active = []
        # Maps Episode objects to InProgress objects for active downloads.
        self._retrieve_tasks = {}
        self._next_episode_check_timer  = None

        # If the config schema changed (new version of Stagehand installed?)
        # then write out a new config file.
        config.save(paths.config, force=not os.path.exists(paths.config))

        # Monitor config file for changes.
        config.watch()
        config.autosave = True
        config.signals['reloaded'].connect(self._load_config)

        self.tvdb = TVDB(paths.db, loop=self.loop)


    @asyncio.coroutine
    def start(self):
        """
        Starts the Stagehand manager, which starts all plugins and
        schedules tasks for
        """
        # TODO: randomize time, twice a day
        self.loop.call_soon(asyncio.async, self._check_update_tvdb())
        #web.notify('alert', title='Global alert', text='Stagehand was restarted')

        yield from self._load_config()
        self._schedule_next_episode_check(skip_current_hour=False)

        # Start all plugins in parallel
        yield from asyncio.gather(searchers.start(self), retrievers.start(self),
                                  notifiers.start(self), providers.start(self))

        # Resume downloading any episodes we aborted by adding to retrieve queue.
        for series in self.tvdb.series:
            for ep in series.episodes:
                if ep.ready and ep.search_result:
                    self._retrieve_queue.append((ep, [ep.search_result]))

        # Start the retrieve queue processor
        self._retrieve_queue_processor()



    @asyncio.coroutine
    def _load_config(self, changed=None):
        if changed is not None:
            # if changed is given (even if it's an empty list), it means we've
            # been invoked from the config reloaded signal.
            log.info('config file changed; reloading')
            if 'retrievers.parallel' in changed:
                # Might have increased the number of parallel downloads allowed.
                # Wake up retrieve queue manager.
                self._retrieve_queue_processor_signal.emit_when_handled()
        yield from self._load_series_from_config()



    def _schedule_next_episode_check(self, skip_current_hour=True):
        try:
            hours = sorted(int(h) for h in config.searchers.hours.split(','))
        except ValueError:
            log.warning('invalid searchers.hours config value (%s), using default', config.searchers.hours)
            hours = sorted(int(h) for h in get_default(config.searchers.hours).split(','))

        # Pick a random minute of the hour for the next check.<commit_msg>Yield frequently during download resumption check at startup<commit_after>import os
import time
import re
import logging
import random
import gc
import asyncio
from datetime import datetime, timedelta

from . import web, searchers, retrievers, notifiers, providers
from .toolbox import FutureProgress, singleton
from .toolbox.utils import fsname
from .toolbox.dateutils import to_timestamp
from .toolbox.config import get_default
from .tvdb import TVDB
from .config import config
from .utils import fixsep
from .searchers import SearcherError
from .retrievers import RetrieverError, RetrieverSoftError, RetrieverHardError, RetrieverAborted, RetrieverAbortedHard, RetrieverAbortedSoft
from .notifiers import NotifierError

log = logging.getLogger('stagehand.manager')


class Manager:
    def __init__(self, paths, *, loop=None):
        super().__init__()
        self.paths = paths
        self.loop = loop or asyncio.get_event_loop()
        self._retrieve_queue_event = asyncio.Event()
        # A list of episodes to be retrieved.
        # [(Episode, [SearchResult, ...]), (Episode, [SearchResult, ...])]
        self._retrieve_queue = []
        # Like _retrieve_queue, but these are the episodes actively being processed.
        # The first SearchResult is the one being retrieved.
        self._retrieve_queue_active = []
        # Maps Episode objects to InProgress objects for active downloads.
        self._retrieve_tasks = {}
        self._next_episode_check_timer  = None

        # If the config schema changed (new version of Stagehand installed?)
        # then write out a new config file.
        config.save(paths.config, force=not os.path.exists(paths.config))

        # Monitor config file for changes.
        config.watch()
        config.autosave = True
        config.signals['reloaded'].connect(self._load_config)

        self.tvdb = TVDB(paths.db, loop=self.loop)


    @asyncio.coroutine
    def start(self):
        """
        Starts the Stagehand manager, which starts all plugins and
        schedules tasks for
        """
        # TODO: randomize time, twice a day
        self.loop.call_soon(asyncio.async, self._check_update_tvdb())
        #web.notify('alert', title='Global alert', text='Stagehand was restarted')

        yield from self._load_config()
        self._schedule_next_episode_check(skip_current_hour=False)

        # Start all plugins in parallel
        yield from asyncio.gather(searchers.start(self), retrievers.start(self),
                                  notifiers.start(self), providers.start(self))

        # Resume downloading any episodes we aborted by adding to retrieve queue.
        log.info('checking all epsiodes to see if any need resuming')
        for series in self.tvdb.series:
            for ep in series.episodes:
                if ep.ready and ep.search_result:
                    self._retrieve_queue.append((ep, [ep.search_result]))
            # For a large number of series, this loop (which involves reading
            # all episodes from the database) can take some time.  So yield
            # back to the main loop so we don't starve other tasks.
            yield

        # Start the retrieve queue processor
        self._retrieve_queue_processor()



    @asyncio.coroutine
    def _load_config(self, changed=None):
        if changed is not None:
            # if changed is given (even if it's an empty list), it means we've
            # been invoked from the config reloaded signal.
            log.info('config file changed; reloading')
            if 'retrievers.parallel' in changed:
                # Might have increased the number of parallel downloads allowed.
                # Wake up retrieve queue manager.
                self._retrieve_queue_processor_signal.emit_when_handled()
        yield from self._load_series_from_config()



    def _schedule_next_episode_check(self, skip_current_hour=True):
        try:
            hours = sorted(int(h) for h in config.searchers.hours.split(','))
        except ValueError:
            log.warning('invalid searchers.hours config value (%s), using default', config.searchers.hours)
            hours = sorted(int(h) for h in get_default(config.searchers.hours).split(','))

        # Pick a random minute of the hour for the next check.
{'fraction_non_alphanumeric': 0.07984467904380536, 'fraction_numerical': 0.0, 'mean_word_length': 2.7260397830018084, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6678307', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2187, 'n_words': 711}