# Path: webcheck/config.py
# BASE_URLS_ONLY = False
# AVOID_EXTERNAL_LINKS = False
# IOTIMEOUT = 10.0
# OUTPUT_DIR = '.'
# CONTINUE = False
# WAIT_BETWEEN_REQUESTS = 0
# MAX_DEPTH = None
# REDIRECT_DEPTH = 5
# PLUGINS = ['webcheck.plugins.anchors',
#            'webcheck.plugins.sitemap',
#            'webcheck.plugins.urllist',
#            'webcheck.plugins.images',
#            'webcheck.plugins.external',
#            'webcheck.plugins.notchkd',
#            'webcheck.plugins.badlinks',
#            'webcheck.plugins.old',
#            'webcheck.plugins.new',
#            'webcheck.plugins.size',
#            'webcheck.plugins.notitles',
#            'webcheck.plugins.problems',
#            'webcheck.plugins.about',
#            'webcheck.plugins.csvfile']
# OVERWRITE_FILES = False
# BYPASSHTTPCACHE = False
# REPORT_SITEMAP_LEVEL = 8
# REPORT_WHATSOLD_URL_AGE = 700
# REPORT_WHATSNEW_URL_AGE = 7
# REPORT_SLOW_URL_SIZE = 76
# PARENT_LISTLEN = 10
# REPORT_LINKS_IN_NEW_WINDOW = False
# USE_ROBOTS = True
# TIDY_OPTIONS = dict(quiet=1,
#                     accessibility_check=1,
#                     show_errors=6,
#                     show_warnings=1,
#                     char_encoding='utf8')
# 
# Path: webcheck/myurllib.py
# def normalizeurl(url):
#     """Return a normalized URL."""
#     return _urlclean(url)


import logging
import urlparse
from sqlalchemy import Table, Column, Integer, Boolean, String, DateTime, ForeignKey
from sqlalchemy import create_engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, backref, sessionmaker
from sqlalchemy.orm.session import object_session
from sqlalchemy.sql.expression import union
from webcheck import config
from webcheck.myurllib import normalizeurl
    # crawling information
    redirectdepth = Column(Integer, default=0)
    depth = Column(Integer, default=0)

    @staticmethod
    def clean_url(url):
        """normalise the URL, removing the fragment from the URL"""
        return urlparse.urldefrag(normalizeurl(url))[0]

    @staticmethod
    def get_or_create(session, url):
        """This expects a clean url."""
        session.commit()
        while True:
            instance = session.query(Link).filter_by(url=url).first()
            if instance:
                return instance
            try:
                instance = Link(url=url)
                session.add(instance)
                session.commit()
                return instance
            except IntegrityError:
                pass  # will try again

    def _get_child(self, url):
        """Get a link object for the specified URL."""
        # get the session
        session = object_session(self)
        # normalise the URL, removing the fragment from the URL

# Path: webcheck/config.py
# BASE_URLS_ONLY = False
# AVOID_EXTERNAL_LINKS = False
# IOTIMEOUT = 10.0
# OUTPUT_DIR = '.'
# CONTINUE = False
# WAIT_BETWEEN_REQUESTS = 0
# MAX_DEPTH = None
# REDIRECT_DEPTH = 5
# PLUGINS = ['webcheck.plugins.anchors',
#            'webcheck.plugins.sitemap',
#            'webcheck.plugins.urllist',
#            'webcheck.plugins.images',
#            'webcheck.plugins.external',
#            'webcheck.plugins.notchkd',
#            'webcheck.plugins.badlinks',
#            'webcheck.plugins.old',
#            'webcheck.plugins.new',
#            'webcheck.plugins.size',
#            'webcheck.plugins.notitles',
#            'webcheck.plugins.problems',
#            'webcheck.plugins.about',
#            'webcheck.plugins.csvfile']
# OVERWRITE_FILES = False
# BYPASSHTTPCACHE = False
# REPORT_SITEMAP_LEVEL = 8
# REPORT_WHATSOLD_URL_AGE = 700
# REPORT_WHATSNEW_URL_AGE = 7
# REPORT_SLOW_URL_SIZE = 76
# PARENT_LISTLEN = 10
# REPORT_LINKS_IN_NEW_WINDOW = False
# USE_ROBOTS = True
# TIDY_OPTIONS = dict(quiet=1,
#                     accessibility_check=1,
#                     show_errors=6,
#                     show_warnings=1,
#                     char_encoding='utf8')
# 
# Path: webcheck/myurllib.py
# def normalizeurl(url):
#     """Return a normalized URL."""
#     return _urlclean(url)


# Path: webcheck/db.py
import logging
import urlparse
from sqlalchemy import Table, Column, Integer, Boolean, String, DateTime, ForeignKey
from sqlalchemy import create_engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, backref, sessionmaker
from sqlalchemy.orm.session import object_session
from sqlalchemy.sql.expression import union
from webcheck import config
from webcheck.myurllib import normalizeurl

    # crawling information
    redirectdepth = Column(Integer, default=0)
    depth = Column(Integer, default=0)

    @staticmethod
    def clean_url(url):
        """normalise the URL, removing the fragment from the URL"""
        return urlparse.urldefrag(normalizeurl(url))[0]

    @staticmethod
    def get_or_create(session, url):
        """This expects a clean url."""
        session.commit()
        while True:
            instance = session.query(Link).filter_by(url=url).first()
            if instance:
                return instance
            try:
                instance = Link(url=url)
                session.add(instance)
                session.commit()
                return instance
            except IntegrityError:
                pass  # will try again

    def _get_child(self, url):
        """Get a link object for the specified URL."""
        # get the session
        session = object_session(self)
        # normalise the URL, removing the fragment from the URL

        url, fragment = urlparse.urldefrag(normalizeurl(url))