# Path: ep_celery.py

# 
# Path: jarr/bootstrap.py
# def is_secure_served():
# def init_logging(log_path=None, log_level=logging.INFO, modules=(),
#                  log_format='%(asctime)s %(levelname)s %(message)s'):
# def init_db(echo=False):
# def init_models():
# def commit_pending_sql(*args, **kwargs):
# def rollback_pending_sql(*args, **kwargs):
# REDIS_CONN = Redis(host=conf.db.redis.host,
#                    db=conf.db.redis.db,
#                    port=conf.db.redis.port)
# 
# Path: jarr/crawler/utils.py
# class Queues(Enum):
#     DEFAULT = conf.celery.task_default_queue
#     CRAWLING = 'jarr-crawling'
#     CLUSTERING = 'jarr-clustering'
# 
# def lock(prefix, expire=LOCK_EXPIRE):
#     def metawrapper(func):
#         @wraps(func)
#         def wrapper(args):
#             start = datetime.now()
#             key = str(args).encode('utf8')
#             key = f"lock-{prefix}-{sha256(key).hexdigest()}"
#             if REDIS_CONN.setnx(key, 'locked'):
#                 REDIS_CONN.expire(key, expire)
#                 try:
#                     return func(args)
#                 except Exception as error:
#                     observe_worker_result_since(start, prefix,
#                                                 error.__class__.__name__)
#                     logger.debug('something wrong happen %r', error)
#                     raise
#                 finally:
#                     observe_worker_result_since(start, prefix, 'ok')
#                     REDIS_CONN.delete(key)
#             else:
#                 observe_worker_result_since(start, prefix, 'skipped')
#         return wrapper
#     return metawrapper
# 
# def observe_worker_result_since(start, method, result):
#     duration = (datetime.now() - start).total_seconds()
#     WORKER.labels(method=method, result=result).observe(duration)
# 
# Path: jarr/lib/enums.py
# class FeedStatus(Enum):
#     active = 'active'
#     paused = 'paused'
#     to_delete = 'to_delete'
#     deleting = 'deleting'
# 
# Path: jarr/lib/utils.py
# def utc_now():
#     return datetime.utcnow().replace(tzinfo=timezone.utc)
# 
# Path: jarr/metrics.py
# ARTICLES = prom(Gauge, 'articles', 'Article counts', ['status'])
# 
# USER = prom(Gauge, 'users', 'User counts', ['status'])
# 
# WORKER_BATCH = prom(Histogram, 'worker_batch', 'worker batch size',
#                     ['worker_type'], buckets=BUCKETS_3H[:-7])


import logging
import urllib3
from datetime import datetime, timedelta
from ep_celery import celery_app
from jarr.bootstrap import REDIS_CONN, conf
from jarr.controllers import (ArticleController, ClusterController,
                              FeedController, UserController)
from jarr.crawler.utils import Queues, lock, observe_worker_result_since
from jarr.lib.enums import FeedStatus
from jarr.lib.utils import utc_now
from jarr.metrics import ARTICLES, USER, WORKER_BATCH
    USER.labels(status='active').set(active.count())


@celery_app.task(name='metrics.users.long_term')
def metrics_users_long_term():
    logger.debug('Counting long term users')
    threshold_connection = utc_now() - timedelta(days=conf.feed.stop_fetch)
    threshold_connection = utc_now() - timedelta(days=conf.feed.stop_fetch)
    threshold_created = utc_now() - timedelta(days=conf.feed.stop_fetch + 1)
    long_term = UserController().read(is_active=True,
                                      last_connection__ge=threshold_connection,
                                      date_created__lt=threshold_created)
    USER.labels(status='long_term').set(long_term.count())


@celery_app.task(name='metrics.articles.unclustered')
def metrics_articles_unclustered():
    logger.debug('Counting unclustered articles')
    unclustered = ArticleController().count_unclustered()
    ARTICLES.labels(status='unclustered').set(unclustered)


@celery_app.task(name='scheduler')
def scheduler():
    logger.warning("Running scheduler")
    start = datetime.now()
    fctrl = FeedController()
    # browsing feeds to fetch
    queue = Queues.CRAWLING if conf.crawler.use_queues else Queues.DEFAULT
    feeds = list(fctrl.list_fetchable(conf.crawler.batch_size))

# Path: ep_celery.py

# 
# Path: jarr/bootstrap.py
# def is_secure_served():
# def init_logging(log_path=None, log_level=logging.INFO, modules=(),
#                  log_format='%(asctime)s %(levelname)s %(message)s'):
# def init_db(echo=False):
# def init_models():
# def commit_pending_sql(*args, **kwargs):
# def rollback_pending_sql(*args, **kwargs):
# REDIS_CONN = Redis(host=conf.db.redis.host,
#                    db=conf.db.redis.db,
#                    port=conf.db.redis.port)
# 
# Path: jarr/crawler/utils.py
# class Queues(Enum):
#     DEFAULT = conf.celery.task_default_queue
#     CRAWLING = 'jarr-crawling'
#     CLUSTERING = 'jarr-clustering'
# 
# def lock(prefix, expire=LOCK_EXPIRE):
#     def metawrapper(func):
#         @wraps(func)
#         def wrapper(args):
#             start = datetime.now()
#             key = str(args).encode('utf8')
#             key = f"lock-{prefix}-{sha256(key).hexdigest()}"
#             if REDIS_CONN.setnx(key, 'locked'):
#                 REDIS_CONN.expire(key, expire)
#                 try:
#                     return func(args)
#                 except Exception as error:
#                     observe_worker_result_since(start, prefix,
#                                                 error.__class__.__name__)
#                     logger.debug('something wrong happen %r', error)
#                     raise
#                 finally:
#                     observe_worker_result_since(start, prefix, 'ok')
#                     REDIS_CONN.delete(key)
#             else:
#                 observe_worker_result_since(start, prefix, 'skipped')
#         return wrapper
#     return metawrapper
# 
# def observe_worker_result_since(start, method, result):
#     duration = (datetime.now() - start).total_seconds()
#     WORKER.labels(method=method, result=result).observe(duration)
# 
# Path: jarr/lib/enums.py
# class FeedStatus(Enum):
#     active = 'active'
#     paused = 'paused'
#     to_delete = 'to_delete'
#     deleting = 'deleting'
# 
# Path: jarr/lib/utils.py
# def utc_now():
#     return datetime.utcnow().replace(tzinfo=timezone.utc)
# 
# Path: jarr/metrics.py
# ARTICLES = prom(Gauge, 'articles', 'Article counts', ['status'])
# 
# USER = prom(Gauge, 'users', 'User counts', ['status'])
# 
# WORKER_BATCH = prom(Histogram, 'worker_batch', 'worker batch size',
#                     ['worker_type'], buckets=BUCKETS_3H[:-7])


# Path: jarr/crawler/main.py
import logging
import urllib3
from datetime import datetime, timedelta
from ep_celery import celery_app
from jarr.bootstrap import REDIS_CONN, conf
from jarr.controllers import (ArticleController, ClusterController,
                              FeedController, UserController)
from jarr.crawler.utils import Queues, lock, observe_worker_result_since
from jarr.lib.enums import FeedStatus
from jarr.lib.utils import utc_now
from jarr.metrics import ARTICLES, USER, WORKER_BATCH

    USER.labels(status='active').set(active.count())


@celery_app.task(name='metrics.users.long_term')
def metrics_users_long_term():
    logger.debug('Counting long term users')
    threshold_connection = utc_now() - timedelta(days=conf.feed.stop_fetch)
    threshold_connection = utc_now() - timedelta(days=conf.feed.stop_fetch)
    threshold_created = utc_now() - timedelta(days=conf.feed.stop_fetch + 1)
    long_term = UserController().read(is_active=True,
                                      last_connection__ge=threshold_connection,
                                      date_created__lt=threshold_created)
    USER.labels(status='long_term').set(long_term.count())


@celery_app.task(name='metrics.articles.unclustered')
def metrics_articles_unclustered():
    logger.debug('Counting unclustered articles')
    unclustered = ArticleController().count_unclustered()
    ARTICLES.labels(status='unclustered').set(unclustered)


@celery_app.task(name='scheduler')
def scheduler():
    logger.warning("Running scheduler")
    start = datetime.now()
    fctrl = FeedController()
    # browsing feeds to fetch
    queue = Queues.CRAWLING if conf.crawler.use_queues else Queues.DEFAULT
    feeds = list(fctrl.list_fetchable(conf.crawler.batch_size))

    WORKER_BATCH.labels(worker_type='fetch-feed').observe(len(feeds))