# Path: jarr/bootstrap.py
# def is_secure_served():
# def init_logging(log_path=None, log_level=logging.INFO, modules=(),
#                  log_format='%(asctime)s %(levelname)s %(message)s'):
# def init_db(echo=False):
# def init_models():
# def commit_pending_sql(*args, **kwargs):
# def rollback_pending_sql(*args, **kwargs):
# REDIS_CONN = Redis(host=conf.db.redis.host,
#                    db=conf.db.redis.db,
#                    port=conf.db.redis.port)
# 
# Path: jarr/controllers/article.py
# class ArticleController(AbstractController):
#     def challenge(self, ids):
#     def _filter_unclustered(*fields):
#     def count_unclustered(cls):
#     def get_user_id_with_pending_articles(cls):
#     def enhance(article):
#     def create(self, **attrs):
#     def update(self, filters, attrs, return_objs=False, commit=True):
#     def remove_from_cluster(self, article):
#     def delete_only_article(article, commit):
#     def delete(self, obj_id, commit=True):
# 
# Path: jarr/lib/enums.py
# class ArticleType(Enum):
#     image = 'image'
#     video = 'video'
#     audio = 'audio'
#     embedded = 'embedded'
# 
# class FeedType(Enum):
#     classic = 'classic'
#     json = 'json'
#     tumblr = 'tumblr'
#     instagram = 'instagram'
#     soundcloud = 'soundcloud'
#     reddit = 'reddit'
#     koreus = 'koreus'
#     twitter = 'twitter'
# 
# Path: jarr/lib/utils.py
# def clean_lang(lang: str):
#     if not lang or not isinstance(lang, str):
#         return
#     if LANG_FORMAT.match(lang):
#         return lang
#     if not CORRECTABLE_LANG_FORMAT.match(lang):
#         return
#     proper_lang = lang[0:2].lower()
#     if len(lang) >= 5:
#         proper_lang = "%s_%s" % (proper_lang, lang[3:5].upper())
#     return proper_lang


import logging
import re
import urllib.parse
from functools import lru_cache
from typing import Optional
from goose3 import Goose
from lxml import etree
from jarr.bootstrap import conf
from jarr.controllers.article import to_vector
from jarr.lib.enums import ArticleType, FeedType
from jarr.lib.url_cleaners import remove_utm_tags
from jarr.lib.utils import clean_lang
                return {'type': 'youtube', 'link': yt_match.group(5)}
            except IndexError:
                pass
        else:
            logger.warning('embedded video not recognized %r',
                           self.article.link)
        return {}


class TruncatedContentGenerator(ContentGenerator):

    def generate(self):
        if self._page is None:
            self._get_goose()
        content = {'type': 'fetched'}
        try:
            content['content'] = self._from_goose_to_html()
            content['link'] = remove_utm_tags(self._page.final_url)
            content['title'] = self._page.title
        except Exception:
            logger.exception("Could not rebuild parsed content for %r",
                             self.article)
            return {}
        if self.article.comments:
            content['comments'] = self.article.comments
        logger.debug('%r no special type found doing nothing', self.article)
        return content


class RedditContentGenerator(TruncatedContentGenerator):

# Path: jarr/bootstrap.py
# def is_secure_served():
# def init_logging(log_path=None, log_level=logging.INFO, modules=(),
#                  log_format='%(asctime)s %(levelname)s %(message)s'):
# def init_db(echo=False):
# def init_models():
# def commit_pending_sql(*args, **kwargs):
# def rollback_pending_sql(*args, **kwargs):
# REDIS_CONN = Redis(host=conf.db.redis.host,
#                    db=conf.db.redis.db,
#                    port=conf.db.redis.port)
# 
# Path: jarr/controllers/article.py
# class ArticleController(AbstractController):
#     def challenge(self, ids):
#     def _filter_unclustered(*fields):
#     def count_unclustered(cls):
#     def get_user_id_with_pending_articles(cls):
#     def enhance(article):
#     def create(self, **attrs):
#     def update(self, filters, attrs, return_objs=False, commit=True):
#     def remove_from_cluster(self, article):
#     def delete_only_article(article, commit):
#     def delete(self, obj_id, commit=True):
# 
# Path: jarr/lib/enums.py
# class ArticleType(Enum):
#     image = 'image'
#     video = 'video'
#     audio = 'audio'
#     embedded = 'embedded'
# 
# class FeedType(Enum):
#     classic = 'classic'
#     json = 'json'
#     tumblr = 'tumblr'
#     instagram = 'instagram'
#     soundcloud = 'soundcloud'
#     reddit = 'reddit'
#     koreus = 'koreus'
#     twitter = 'twitter'
# 
# Path: jarr/lib/utils.py
# def clean_lang(lang: str):
#     if not lang or not isinstance(lang, str):
#         return
#     if LANG_FORMAT.match(lang):
#         return lang
#     if not CORRECTABLE_LANG_FORMAT.match(lang):
#         return
#     proper_lang = lang[0:2].lower()
#     if len(lang) >= 5:
#         proper_lang = "%s_%s" % (proper_lang, lang[3:5].upper())
#     return proper_lang


# Path: jarr/lib/content_generator.py
import logging
import re
import urllib.parse
from functools import lru_cache
from typing import Optional
from goose3 import Goose
from lxml import etree
from jarr.bootstrap import conf
from jarr.controllers.article import to_vector
from jarr.lib.enums import ArticleType, FeedType
from jarr.lib.url_cleaners import remove_utm_tags
from jarr.lib.utils import clean_lang

                return {'type': 'youtube', 'link': yt_match.group(5)}
            except IndexError:
                pass
        else:
            logger.warning('embedded video not recognized %r',
                           self.article.link)
        return {}


class TruncatedContentGenerator(ContentGenerator):

    def generate(self):
        if self._page is None:
            self._get_goose()
        content = {'type': 'fetched'}
        try:
            content['content'] = self._from_goose_to_html()
            content['link'] = remove_utm_tags(self._page.final_url)
            content['title'] = self._page.title
        except Exception:
            logger.exception("Could not rebuild parsed content for %r",
                             self.article)
            return {}
        if self.article.comments:
            content['comments'] = self.article.comments
        logger.debug('%r no special type found doing nothing', self.article)
        return content


class RedditContentGenerator(TruncatedContentGenerator):

    feed_type = FeedType.reddit