# Path: cliche/celery.py
# class Loader(BaseLoader):
#     def read_configuration(self):
# def get_database_engine() -> Engine:
# def get_session() -> Session:
# def close_session(task_id, task, *args, **kwargs):
# def get_raven_client() -> Client:
# def setup_raven_logging(conf=None, **kwargs):
# def report_task_failure(task_id, exception, args, kwargs,
#                         traceback, einfo, sender):


import datetime
import urllib.parse
import requests
from celery.utils.log import get_task_logger
from lxml.html import document_fromstring, parse
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import FlushError, NoResultFound
from ...celery import app, get_session
from .entities import Entity, Redirection, Relation
        *namespace, name = name.split(':')
        name = name.strip()
        namespace = 'Main' if not namespace else namespace[0]
        type = determine_type(namespace)
        if type == 'Administrivia':
            return False, tree, namespace, name, final_url
        upsert_entity(session, namespace, name, type, final_url)
        process_redirections(session, url, final_url, namespace, name)
        return True, tree, namespace, name, final_url


def recently_crawled(current_time, url, session):
    logger = get_task_logger(__name__ + '.recently_crawled')
    try:
        last_crawled = session.query(Entity).filter_by(url=url) \
                              .one().last_crawled
        if last_crawled:
            if current_time - last_crawled < CRAWL_INTERVAL:
                logger.info('%s was recently crawled in %s days.',
                            url, CRAWL_INTERVAL)
                return True
    except NoResultFound:
        pass
    return False


def is_wiki_page(url):
    return (BASE_URL not in url or WIKI_PAGE in url)



# Path: cliche/celery.py
# class Loader(BaseLoader):
#     def read_configuration(self):
# def get_database_engine() -> Engine:
# def get_session() -> Session:
# def close_session(task_id, task, *args, **kwargs):
# def get_raven_client() -> Client:
# def setup_raven_logging(conf=None, **kwargs):
# def report_task_failure(task_id, exception, args, kwargs,
#                         traceback, einfo, sender):


# Path: cliche/services/tvtropes/crawler.py
import datetime
import urllib.parse
import requests
from celery.utils.log import get_task_logger
from lxml.html import document_fromstring, parse
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import FlushError, NoResultFound
from ...celery import app, get_session
from .entities import Entity, Redirection, Relation

        *namespace, name = name.split(':')
        name = name.strip()
        namespace = 'Main' if not namespace else namespace[0]
        type = determine_type(namespace)
        if type == 'Administrivia':
            return False, tree, namespace, name, final_url
        upsert_entity(session, namespace, name, type, final_url)
        process_redirections(session, url, final_url, namespace, name)
        return True, tree, namespace, name, final_url


def recently_crawled(current_time, url, session):
    logger = get_task_logger(__name__ + '.recently_crawled')
    try:
        last_crawled = session.query(Entity).filter_by(url=url) \
                              .one().last_crawled
        if last_crawled:
            if current_time - last_crawled < CRAWL_INTERVAL:
                logger.info('%s was recently crawled in %s days.',
                            url, CRAWL_INTERVAL)
                return True
    except NoResultFound:
        pass
    return False


def is_wiki_page(url):
    return (BASE_URL not in url or WIKI_PAGE in url)



@app.task