# Path: jarr/crawler/main.py
# @celery_app.task(name='scheduler')
# def scheduler():
#     logger.warning("Running scheduler")
#     start = datetime.now()
#     fctrl = FeedController()
#     # browsing feeds to fetch
#     queue = Queues.CRAWLING if conf.crawler.use_queues else Queues.DEFAULT
#     feeds = list(fctrl.list_fetchable(conf.crawler.batch_size))
#     WORKER_BATCH.labels(worker_type='fetch-feed').observe(len(feeds))
#     logger.info('%d to enqueue', len(feeds))
#     for feed in feeds:
#         logger.debug("%r: scheduling to be fetched on queue:%r",
#                      feed, queue.value)
#         process_feed.apply_async(args=[feed.id], queue=queue.value)
#     # browsing feeds to delete
#     feeds_to_delete = list(fctrl.read(status=FeedStatus.to_delete))
#     if feeds_to_delete and REDIS_CONN.setnx(JARR_FEED_DEL_KEY, 'true'):
#         REDIS_CONN.expire(JARR_FEED_DEL_KEY, LOCK_EXPIRE)
#         logger.info('%d to delete, deleting one', len(feeds_to_delete))
#         for feed in feeds_to_delete:
#             logger.debug("%r: scheduling to be delete", feed)
#             feed_cleaner.apply_async(args=[feed.id])
#     # applying clusterizer
#     queue = Queues.CLUSTERING if conf.crawler.use_queues else Queues.DEFAULT
#     for user_id in ArticleController.get_user_id_with_pending_articles():
#         if REDIS_CONN.setnx(JARR_CLUSTERIZER_KEY % user_id, 'true'):
#             REDIS_CONN.expire(JARR_CLUSTERIZER_KEY % user_id,
#                               conf.crawler.clusterizer_delay)
#             logger.debug('Scheduling clusterizer for User(%d) on queue:%r',
#                          user_id, queue.value)
#             clusterizer.apply_async(args=[user_id], queue=queue.value)
#     scheduler.apply_async(countdown=conf.crawler.idle_delay)
#     metrics_users_any.apply_async()
#     metrics_users_active.apply_async()
#     metrics_users_long_term.apply_async()
#     metrics_articles_unclustered.apply_async()
#     observe_worker_result_since(start, 'scheduler', 'ok')
# 
# Path: jarr/lib/utils.py
# def utc_now():
#     return datetime.utcnow().replace(tzinfo=timezone.utc)


from datetime import datetime, timezone
from mock import patch
from jarr.controllers import FeedController, UserController
from jarr.crawler.main import scheduler
from jarr.lib.utils import utc_now
from tests.base import BaseJarrTest
        self.process_feed_patch = self._process_feed_patch.start()
        self.feed_cleaner_patch = self._feed_cleaner_patch.start()
        self.scheduler_patch = self._sched_async.start()
        for metrics_patch in self._metrics:
            metrics_patch.start()

    def tearDown(self):
        self._clusteriser_patch.stop()
        self._process_feed_patch.stop()
        self._feed_cleaner_patch.stop()
        self._sched_async.stop()
        for metrics_patch in self._metrics:
            metrics_patch.stop()

    def test_scheduler(self):
        scheduler()
        UserController().update({}, {'last_connection': utc_now()})
        fctrl = FeedController()

        epoch = datetime(1970, 1, 1, tzinfo=timezone.utc)
        self.assertEqual(fctrl.read().count(),
                         self.process_feed_patch.apply_async.call_count)
        self.assertEqual(0, self.clusteriser_patch.apply_async.call_count)
        self.assertEqual(0, self.feed_cleaner_patch.apply_async.call_count)
        feed1, feed2, feed3 = list(FeedController().read().limit(3))
        FeedController().update({'id__in': [feed1.id, feed3.id]},
                                {'status': 'to_delete'})
        FeedController().update({'id': feed2.id},
                                {'last_retrieved': epoch, 'expires': epoch})
        self.assertEqual(1, len(list(fctrl.list_fetchable())))

# Path: jarr/crawler/main.py
# @celery_app.task(name='scheduler')
# def scheduler():
#     logger.warning("Running scheduler")
#     start = datetime.now()
#     fctrl = FeedController()
#     # browsing feeds to fetch
#     queue = Queues.CRAWLING if conf.crawler.use_queues else Queues.DEFAULT
#     feeds = list(fctrl.list_fetchable(conf.crawler.batch_size))
#     WORKER_BATCH.labels(worker_type='fetch-feed').observe(len(feeds))
#     logger.info('%d to enqueue', len(feeds))
#     for feed in feeds:
#         logger.debug("%r: scheduling to be fetched on queue:%r",
#                      feed, queue.value)
#         process_feed.apply_async(args=[feed.id], queue=queue.value)
#     # browsing feeds to delete
#     feeds_to_delete = list(fctrl.read(status=FeedStatus.to_delete))
#     if feeds_to_delete and REDIS_CONN.setnx(JARR_FEED_DEL_KEY, 'true'):
#         REDIS_CONN.expire(JARR_FEED_DEL_KEY, LOCK_EXPIRE)
#         logger.info('%d to delete, deleting one', len(feeds_to_delete))
#         for feed in feeds_to_delete:
#             logger.debug("%r: scheduling to be delete", feed)
#             feed_cleaner.apply_async(args=[feed.id])
#     # applying clusterizer
#     queue = Queues.CLUSTERING if conf.crawler.use_queues else Queues.DEFAULT
#     for user_id in ArticleController.get_user_id_with_pending_articles():
#         if REDIS_CONN.setnx(JARR_CLUSTERIZER_KEY % user_id, 'true'):
#             REDIS_CONN.expire(JARR_CLUSTERIZER_KEY % user_id,
#                               conf.crawler.clusterizer_delay)
#             logger.debug('Scheduling clusterizer for User(%d) on queue:%r',
#                          user_id, queue.value)
#             clusterizer.apply_async(args=[user_id], queue=queue.value)
#     scheduler.apply_async(countdown=conf.crawler.idle_delay)
#     metrics_users_any.apply_async()
#     metrics_users_active.apply_async()
#     metrics_users_long_term.apply_async()
#     metrics_articles_unclustered.apply_async()
#     observe_worker_result_since(start, 'scheduler', 'ok')
# 
# Path: jarr/lib/utils.py
# def utc_now():
#     return datetime.utcnow().replace(tzinfo=timezone.utc)


# Path: tests/crawler/main_test.py
from datetime import datetime, timezone
from mock import patch
from jarr.controllers import FeedController, UserController
from jarr.crawler.main import scheduler
from jarr.lib.utils import utc_now
from tests.base import BaseJarrTest

        self.process_feed_patch = self._process_feed_patch.start()
        self.feed_cleaner_patch = self._feed_cleaner_patch.start()
        self.scheduler_patch = self._sched_async.start()
        for metrics_patch in self._metrics:
            metrics_patch.start()

    def tearDown(self):
        self._clusteriser_patch.stop()
        self._process_feed_patch.stop()
        self._feed_cleaner_patch.stop()
        self._sched_async.stop()
        for metrics_patch in self._metrics:
            metrics_patch.stop()

    def test_scheduler(self):
        scheduler()
        UserController().update({}, {'last_connection': utc_now()})
        fctrl = FeedController()

        epoch = datetime(1970, 1, 1, tzinfo=timezone.utc)
        self.assertEqual(fctrl.read().count(),
                         self.process_feed_patch.apply_async.call_count)
        self.assertEqual(0, self.clusteriser_patch.apply_async.call_count)
        self.assertEqual(0, self.feed_cleaner_patch.apply_async.call_count)
        feed1, feed2, feed3 = list(FeedController().read().limit(3))
        FeedController().update({'id__in': [feed1.id, feed3.id]},
                                {'status': 'to_delete'})
        FeedController().update({'id': feed2.id},
                                {'last_retrieved': epoch, 'expires': epoch})
        self.assertEqual(1, len(list(fctrl.list_fetchable())))

        scheduler()