# Path: staticgen/conf.py
# class StaticgenConf(AppConf):
#     class Meta:
#     SITEMAP_URL = None
#     FAIL_SILENTLY = True
#     MULTITHREAD = True
#     BUCKET_NAME = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', '')
#     STATIC_SITE_DOMAIN = None
# 
# Path: staticgen/exceptions.py
# class StaticgenError(Exception):
#     pass
# 
# Path: staticgen/helpers.py
# def get_static_site_domain():
#     static_site_domain = settings.STATICGEN_STATIC_SITE_DOMAIN
#     if static_site_domain is None:
#         connection = connect_s3(
#             settings.AWS_ACCESS_KEY_ID,
#             settings.AWS_SECRET_ACCESS_KEY
#         )
#         bucket = connection.get_bucket(settings.AWS_STORAGE_BUCKET_NAME)
#         static_site_domain = bucket.get_website_endpoint()
#     return static_site_domain.rstrip('/')


import logging
from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse, NoReverseMatch
from django.utils.translation import ugettext_lazy as _
from django.test import Client
from bs4 import BeautifulSoup
from .conf import settings
from .exceptions import StaticgenError
from .helpers import get_static_site_domain
from .status import is_redirect, is_success
    from urllib.parse import urlparse
    from urlparse import urlparse
    def enqueue(self, url):
        if url and url not in self.visited:
            self.to_visit.add(url)

    def __iter__(self):
        while self.to_visit:
            url = self.to_visit.pop()
            self.visited.add(url)
            yield url


class StaticgenCrawler(object):

    def __init__(self):
        current_site = Site.objects.get_current()
        self.base_domain = current_site.domain
        self.url_registry = UrlRegistry()
        self.client = Client(SERVER_NAME=get_static_site_domain())

    def log_error(self, message):
        logger.error(message)
        if not settings.STATICGEN_FAIL_SILENTLY:
            raise StaticgenError(message)

    def clean_url(self, url):
        url = url.lower().strip()

        special_links = [
            url.startswith(prefix) for prefix in ('mailto', 'tel', 'sms', 'skype', 'geo')]


# Path: staticgen/conf.py
# class StaticgenConf(AppConf):
#     class Meta:
#     SITEMAP_URL = None
#     FAIL_SILENTLY = True
#     MULTITHREAD = True
#     BUCKET_NAME = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', '')
#     STATIC_SITE_DOMAIN = None
# 
# Path: staticgen/exceptions.py
# class StaticgenError(Exception):
#     pass
# 
# Path: staticgen/helpers.py
# def get_static_site_domain():
#     static_site_domain = settings.STATICGEN_STATIC_SITE_DOMAIN
#     if static_site_domain is None:
#         connection = connect_s3(
#             settings.AWS_ACCESS_KEY_ID,
#             settings.AWS_SECRET_ACCESS_KEY
#         )
#         bucket = connection.get_bucket(settings.AWS_STORAGE_BUCKET_NAME)
#         static_site_domain = bucket.get_website_endpoint()
#     return static_site_domain.rstrip('/')


# Path: staticgen/staticgen_crawler.py
import logging
from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse, NoReverseMatch
from django.utils.translation import ugettext_lazy as _
from django.test import Client
from bs4 import BeautifulSoup
from .conf import settings
from .exceptions import StaticgenError
from .helpers import get_static_site_domain
from .status import is_redirect, is_success
    from urllib.parse import urlparse
    from urlparse import urlparse

    def enqueue(self, url):
        if url and url not in self.visited:
            self.to_visit.add(url)

    def __iter__(self):
        while self.to_visit:
            url = self.to_visit.pop()
            self.visited.add(url)
            yield url


class StaticgenCrawler(object):

    def __init__(self):
        current_site = Site.objects.get_current()
        self.base_domain = current_site.domain
        self.url_registry = UrlRegistry()
        self.client = Client(SERVER_NAME=get_static_site_domain())

    def log_error(self, message):
        logger.error(message)
        if not settings.STATICGEN_FAIL_SILENTLY:
            raise StaticgenError(message)

    def clean_url(self, url):
        url = url.lower().strip()

        special_links = [
            url.startswith(prefix) for prefix in ('mailto', 'tel', 'sms', 'skype', 'geo')]


        if any([url.startswith(settings.MEDIA_URL),  # skip media files urls