# Path: jarr/bootstrap.py
# def is_secure_served():
# def init_logging(log_path=None, log_level=logging.INFO, modules=(),
#                  log_format='%(asctime)s %(levelname)s %(message)s'):
# def init_db(echo=False):
# def init_models():
# def commit_pending_sql(*args, **kwargs):
# def rollback_pending_sql(*args, **kwargs):
# REDIS_CONN = Redis(host=conf.db.redis.host,
#                    db=conf.db.redis.db,
#                    port=conf.db.redis.port)
# 
# Path: jarr/crawler/requests_utils.py
# def response_calculated_etag_match(feed, resp):
#     if f'jarr/"{digest(resp.text)}"' == feed.etag:
#         logger.info("%r: calculated hash matches (%d)",
#                     feed, resp.status_code)
#         return True
#     return False
# 
# def response_etag_match(feed, resp):
#     if feed.etag and resp.headers.get('etag'):
#         if feed.etag.startswith('jarr/'):
#             return False  # it's a jarr generated etag
#         if resp.headers['etag'] == feed.etag:
#             logger.info("%r: responded with same etag (%d)",
#                         feed, resp.status_code)
#             return True
#     return False
# 
# Path: jarr/lib/enums.py
# class FeedType(Enum):
#     classic = 'classic'
#     json = 'json'
#     tumblr = 'tumblr'
#     instagram = 'instagram'
#     soundcloud = 'soundcloud'
#     reddit = 'reddit'
#     koreus = 'koreus'
#     twitter = 'twitter'
# 
# Path: jarr/lib/utils.py
# def jarr_get(url, timeout=None, user_agent=None, headers=None, **kwargs):
#     from jarr.bootstrap import conf  # circular import otherwise
#     timeout = timeout or conf.crawler.timeout
#     user_agent = user_agent or conf.crawler.user_agent
#     def_headers = {'User-Agent': user_agent}
#     if headers is not None:
#         def_headers.update(headers)
#     request_kwargs = {'verify': False, 'allow_redirects': True,
#                       'timeout': timeout, 'headers': def_headers}
#     request_kwargs.update(kwargs)
#     return requests.get(url, **request_kwargs)
# 
# def utc_now():
#     return datetime.utcnow().replace(tzinfo=timezone.utc)
# 
# Path: jarr/metrics.py
# FEED_FETCH = prom(Counter, 'feed_fetch', 'Feed fetching event',
#                   ['feed_type', 'result'])


import logging
from typing import Optional
from jarr.bootstrap import conf
from jarr.controllers import ArticleController, FeedController
from jarr.crawler.article_builders.classic import ClassicArticleBuilder
from jarr.crawler.lib.headers_handling import (extract_feed_info,
                                               prepare_headers)
from jarr.crawler.requests_utils import (response_calculated_etag_match,
                                         response_etag_match)
from jarr.lib.enums import FeedType
from jarr.lib.utils import jarr_get, utc_now
from jarr.metrics import FEED_FETCH

    def _metric_fetch(self, result, level=logging.INFO):
        logger.log(level, '%r: responded with %s', self.feed, result)
        FEED_FETCH.labels(feed_type=self.feed.feed_type.value,
                          result=result).inc()

    def set_feed_error(self, error=None, parsed_feed=None):
        error_count = self.feed.error_count + 1
        if error:
            last_error = str(error)
        elif parsed_feed:
            last_error = str(parsed_feed.get('bozo_exception', ''))
        if self.feed.error_count > conf.feed.error_threshold:
            level = logging.WARNING
        else:
            level = logging.DEBUG
        logger.log(level, "%r: fetching feed error'd; error count -> %r",
                        self.feed, error_count)
        logger.debug("%r: last error details %r", self.feed, last_error)
        now = utc_now()
        info = {'error_count': error_count, 'last_error': last_error,
                'user_id': self.feed.user_id, 'last_retrieved': now,
                'expires': None}  # forcing compute by controller

        FEED_FETCH.labels(feed_type=self.feed.feed_type.value,
                          result='error').inc()
        return FeedController().update({'id': self.feed.id}, info)

    def clean_feed(self, response, **info):
        """Will reset the errors counters on a feed that have known errors"""

# Path: jarr/bootstrap.py
# def is_secure_served():
# def init_logging(log_path=None, log_level=logging.INFO, modules=(),
#                  log_format='%(asctime)s %(levelname)s %(message)s'):
# def init_db(echo=False):
# def init_models():
# def commit_pending_sql(*args, **kwargs):
# def rollback_pending_sql(*args, **kwargs):
# REDIS_CONN = Redis(host=conf.db.redis.host,
#                    db=conf.db.redis.db,
#                    port=conf.db.redis.port)
# 
# Path: jarr/crawler/requests_utils.py
# def response_calculated_etag_match(feed, resp):
#     if f'jarr/"{digest(resp.text)}"' == feed.etag:
#         logger.info("%r: calculated hash matches (%d)",
#                     feed, resp.status_code)
#         return True
#     return False
# 
# def response_etag_match(feed, resp):
#     if feed.etag and resp.headers.get('etag'):
#         if feed.etag.startswith('jarr/'):
#             return False  # it's a jarr generated etag
#         if resp.headers['etag'] == feed.etag:
#             logger.info("%r: responded with same etag (%d)",
#                         feed, resp.status_code)
#             return True
#     return False
# 
# Path: jarr/lib/enums.py
# class FeedType(Enum):
#     classic = 'classic'
#     json = 'json'
#     tumblr = 'tumblr'
#     instagram = 'instagram'
#     soundcloud = 'soundcloud'
#     reddit = 'reddit'
#     koreus = 'koreus'
#     twitter = 'twitter'
# 
# Path: jarr/lib/utils.py
# def jarr_get(url, timeout=None, user_agent=None, headers=None, **kwargs):
#     from jarr.bootstrap import conf  # circular import otherwise
#     timeout = timeout or conf.crawler.timeout
#     user_agent = user_agent or conf.crawler.user_agent
#     def_headers = {'User-Agent': user_agent}
#     if headers is not None:
#         def_headers.update(headers)
#     request_kwargs = {'verify': False, 'allow_redirects': True,
#                       'timeout': timeout, 'headers': def_headers}
#     request_kwargs.update(kwargs)
#     return requests.get(url, **request_kwargs)
# 
# def utc_now():
#     return datetime.utcnow().replace(tzinfo=timezone.utc)
# 
# Path: jarr/metrics.py
# FEED_FETCH = prom(Counter, 'feed_fetch', 'Feed fetching event',
#                   ['feed_type', 'result'])


# Path: jarr/crawler/crawlers/abstract.py
import logging
from typing import Optional
from jarr.bootstrap import conf
from jarr.controllers import ArticleController, FeedController
from jarr.crawler.article_builders.classic import ClassicArticleBuilder
from jarr.crawler.lib.headers_handling import (extract_feed_info,
                                               prepare_headers)
from jarr.crawler.requests_utils import (response_calculated_etag_match,
                                         response_etag_match)
from jarr.lib.enums import FeedType
from jarr.lib.utils import jarr_get, utc_now
from jarr.metrics import FEED_FETCH


    def _metric_fetch(self, result, level=logging.INFO):
        logger.log(level, '%r: responded with %s', self.feed, result)
        FEED_FETCH.labels(feed_type=self.feed.feed_type.value,
                          result=result).inc()

    def set_feed_error(self, error=None, parsed_feed=None):
        error_count = self.feed.error_count + 1
        if error:
            last_error = str(error)
        elif parsed_feed:
            last_error = str(parsed_feed.get('bozo_exception', ''))
        if self.feed.error_count > conf.feed.error_threshold:
            level = logging.WARNING
        else:
            level = logging.DEBUG
        logger.log(level, "%r: fetching feed error'd; error count -> %r",
                        self.feed, error_count)
        logger.debug("%r: last error details %r", self.feed, last_error)
        now = utc_now()
        info = {'error_count': error_count, 'last_error': last_error,
                'user_id': self.feed.user_id, 'last_retrieved': now,
                'expires': None}  # forcing compute by controller

        FEED_FETCH.labels(feed_type=self.feed.feed_type.value,
                          result='error').inc()
        return FeedController().update({'id': self.feed.id}, info)

    def clean_feed(self, response, **info):
        """Will reset the errors counters on a feed that have known errors"""

        now = utc_now()