# Path: webcheck/config.py
# BASE_URLS_ONLY = False
# AVOID_EXTERNAL_LINKS = False
# IOTIMEOUT = 10.0
# OUTPUT_DIR = '.'
# CONTINUE = False
# WAIT_BETWEEN_REQUESTS = 0
# MAX_DEPTH = None
# REDIRECT_DEPTH = 5
# PLUGINS = ['webcheck.plugins.anchors',
#            'webcheck.plugins.sitemap',
#            'webcheck.plugins.urllist',
#            'webcheck.plugins.images',
#            'webcheck.plugins.external',
#            'webcheck.plugins.notchkd',
#            'webcheck.plugins.badlinks',
#            'webcheck.plugins.old',
#            'webcheck.plugins.new',
#            'webcheck.plugins.size',
#            'webcheck.plugins.notitles',
#            'webcheck.plugins.problems',
#            'webcheck.plugins.about',
#            'webcheck.plugins.csvfile']
# OVERWRITE_FILES = False
# BYPASSHTTPCACHE = False
# REPORT_SITEMAP_LEVEL = 8
# REPORT_WHATSOLD_URL_AGE = 700
# REPORT_WHATSNEW_URL_AGE = 7
# REPORT_SLOW_URL_SIZE = 76
# PARENT_LISTLEN = 10
# REPORT_LINKS_IN_NEW_WINDOW = False
# USE_ROBOTS = True
# TIDY_OPTIONS = dict(quiet=1,
#                     accessibility_check=1,
#                     show_errors=6,
#                     show_warnings=1,
#                     char_encoding='utf8')
# 
# Path: webcheck/db.py
# class Link(Base):
# class LinkProblem(Base):
# class PageProblem(Base):
# class Anchor(Base):
# class RequestedAnchor(Base):
#     def clean_url(url):
#     def get_or_create(session, url):
#     def _get_child(self, url):
#     def _mk_unicode(self, message):
#     def set_encoding(self, encoding):
#     def add_redirect(self, url):
#     def add_linkproblem(self, message):
#     def add_pageproblem(self, message):
#     def add_child(self, url):
#     def add_embed(self, url):
#     def add_anchor(self, anchor):
#     def add_reqanchor(self, parent, anchor):
#     def follow_link(self, visited=None):
#     def count_parents(self):
#     def parents(self):
#     def __unicode__(self):
#     def __unicode__(self):
#     def __unicode__(self):
#     def __unicode__(self):
# def setup_db(filename):
# def truncate_db():
# 
# Path: webcheck/output.py
# def install_file(source, is_text=False):
#     """Install the given file in the output directory.
#     If the is_text flag is set to true it is assumed the file is text,
#     translating line endings."""
#     sfp = pkg_resources.resource_stream(__name__, source)
#     if is_text:
#         sfp = codecs.getreader('utf-8')(sfp)
#     # TODO: support more schemes here
#     # figure out the destination name
#     target = os.path.join(config.OUTPUT_DIR, os.path.basename(source))
#     # create file in output directory (with overwrite question)
#     tfp = open_file(os.path.basename(source), is_text=is_text)
#     # copy contents
#     shutil.copyfileobj(sfp, tfp)
#     # close files
#     tfp.close()
#     sfp.close()


import atexit
import cookielib
import datetime
import httplib
import logging
import os
import re
import robotparser
import time
import urllib
import urllib2
import urlparse
import webcheck.parsers
    import webcheck  # local import to avoid import loop
from webcheck import config
from webcheck.db import Session, Link, setup_db, truncate_db
from webcheck.output import install_file
        while count > 0:
            logger.debug('%d links at depth %d%s', count, depth,
                         ' (max)' if depth == config.MAX_DEPTH else '')
            # update the depth of all links without a depth that have a
            # parent with the previous depth
            qry = session.query(Link).filter(Link.depth == None)
            qry = qry.filter(Link.linked_from.any(Link.depth == depth))
            count = qry.update(dict(depth=depth + 1), synchronize_session=False)
            session.commit()
            depth += 1
            # TODO: also handle embeds
        session.commit()
        session.close()
        # see if any of the plugins want to do postprocessing
        for plugin in self.plugins:
            if hasattr(plugin, 'postprocess'):
                logger.info(plugin.__name__)
                plugin.postprocess(self)

    def generate(self):
        """Generate pages for plugins."""
        # ensure we have a connection to the database
        self.setup_database()
        # call all the plugins
        for plugin in self.plugins:
            if hasattr(plugin, 'generate'):
                logger.info(plugin.__name__)
                plugin.generate(self)
        # install theme files
        install_file('static/webcheck.css', True)

# Path: webcheck/config.py
# BASE_URLS_ONLY = False
# AVOID_EXTERNAL_LINKS = False
# IOTIMEOUT = 10.0
# OUTPUT_DIR = '.'
# CONTINUE = False
# WAIT_BETWEEN_REQUESTS = 0
# MAX_DEPTH = None
# REDIRECT_DEPTH = 5
# PLUGINS = ['webcheck.plugins.anchors',
#            'webcheck.plugins.sitemap',
#            'webcheck.plugins.urllist',
#            'webcheck.plugins.images',
#            'webcheck.plugins.external',
#            'webcheck.plugins.notchkd',
#            'webcheck.plugins.badlinks',
#            'webcheck.plugins.old',
#            'webcheck.plugins.new',
#            'webcheck.plugins.size',
#            'webcheck.plugins.notitles',
#            'webcheck.plugins.problems',
#            'webcheck.plugins.about',
#            'webcheck.plugins.csvfile']
# OVERWRITE_FILES = False
# BYPASSHTTPCACHE = False
# REPORT_SITEMAP_LEVEL = 8
# REPORT_WHATSOLD_URL_AGE = 700
# REPORT_WHATSNEW_URL_AGE = 7
# REPORT_SLOW_URL_SIZE = 76
# PARENT_LISTLEN = 10
# REPORT_LINKS_IN_NEW_WINDOW = False
# USE_ROBOTS = True
# TIDY_OPTIONS = dict(quiet=1,
#                     accessibility_check=1,
#                     show_errors=6,
#                     show_warnings=1,
#                     char_encoding='utf8')
# 
# Path: webcheck/db.py
# class Link(Base):
# class LinkProblem(Base):
# class PageProblem(Base):
# class Anchor(Base):
# class RequestedAnchor(Base):
#     def clean_url(url):
#     def get_or_create(session, url):
#     def _get_child(self, url):
#     def _mk_unicode(self, message):
#     def set_encoding(self, encoding):
#     def add_redirect(self, url):
#     def add_linkproblem(self, message):
#     def add_pageproblem(self, message):
#     def add_child(self, url):
#     def add_embed(self, url):
#     def add_anchor(self, anchor):
#     def add_reqanchor(self, parent, anchor):
#     def follow_link(self, visited=None):
#     def count_parents(self):
#     def parents(self):
#     def __unicode__(self):
#     def __unicode__(self):
#     def __unicode__(self):
#     def __unicode__(self):
# def setup_db(filename):
# def truncate_db():
# 
# Path: webcheck/output.py
# def install_file(source, is_text=False):
#     """Install the given file in the output directory.
#     If the is_text flag is set to true it is assumed the file is text,
#     translating line endings."""
#     sfp = pkg_resources.resource_stream(__name__, source)
#     if is_text:
#         sfp = codecs.getreader('utf-8')(sfp)
#     # TODO: support more schemes here
#     # figure out the destination name
#     target = os.path.join(config.OUTPUT_DIR, os.path.basename(source))
#     # create file in output directory (with overwrite question)
#     tfp = open_file(os.path.basename(source), is_text=is_text)
#     # copy contents
#     shutil.copyfileobj(sfp, tfp)
#     # close files
#     tfp.close()
#     sfp.close()


# Path: webcheck/crawler.py
import atexit
import cookielib
import datetime
import httplib
import logging
import os
import re
import robotparser
import time
import urllib
import urllib2
import urlparse
import webcheck.parsers
    import webcheck  # local import to avoid import loop
from webcheck import config
from webcheck.db import Session, Link, setup_db, truncate_db
from webcheck.output import install_file

        while count > 0:
            logger.debug('%d links at depth %d%s', count, depth,
                         ' (max)' if depth == config.MAX_DEPTH else '')
            # update the depth of all links without a depth that have a
            # parent with the previous depth
            qry = session.query(Link).filter(Link.depth == None)
            qry = qry.filter(Link.linked_from.any(Link.depth == depth))
            count = qry.update(dict(depth=depth + 1), synchronize_session=False)
            session.commit()
            depth += 1
            # TODO: also handle embeds
        session.commit()
        session.close()
        # see if any of the plugins want to do postprocessing
        for plugin in self.plugins:
            if hasattr(plugin, 'postprocess'):
                logger.info(plugin.__name__)
                plugin.postprocess(self)

    def generate(self):
        """Generate pages for plugins."""
        # ensure we have a connection to the database
        self.setup_database()
        # call all the plugins
        for plugin in self.plugins:
            if hasattr(plugin, 'generate'):
                logger.info(plugin.__name__)
                plugin.generate(self)
        # install theme files
        install_file('static/webcheck.css', True)

        install_file('static/fancytooltips/fancytooltips.js', True)