# Path: staticgen/exceptions.py
# class StaticgenError(Exception):
#     pass
# 
# Path: staticgen/staticgen_crawler.py
# class StaticgenCrawler(object):
# 
#     def __init__(self):
#         current_site = Site.objects.get_current()
#         self.base_domain = current_site.domain
#         self.url_registry = UrlRegistry()
#         self.client = Client(SERVER_NAME=get_static_site_domain())
# 
#     def log_error(self, message):
#         logger.error(message)
#         if not settings.STATICGEN_FAIL_SILENTLY:
#             raise StaticgenError(message)
# 
#     def clean_url(self, url):
#         url = url.lower().strip()
# 
#         special_links = [
#             url.startswith(prefix) for prefix in ('mailto', 'tel', 'sms', 'skype', 'geo')]
# 
#         if any([url.startswith(settings.MEDIA_URL),  # skip media files urls
#                 url.startswith(settings.STATIC_URL),  # skip static files urls
#                 url.startswith('#')] + special_links):  # skip anchors & special links e.g 'tel:'
#             return None
# 
#         if url.startswith('/'):
#             return url
# 
#         # strip web prefixes
#         for prefix in ('http://', 'https://', 'www.', ):
#             if url.startswith(prefix):
#                 url = url[len(prefix):]
# 
#         if not url.startswith(self.base_domain):
#             return None
# 
#         return '//{url}'.format(url=url)
# 
#     def get_sitemap_url(self):
#         sitemap_url = settings.STATICGEN_SITEMAP_URL
#         if sitemap_url is None:
#             try:
#                 # Try for the "global" sitemap URL.
#                 sitemap_url = reverse('django.contrib.sitemaps.views.sitemap')
#             except NoReverseMatch:
#                 message = _('You did not provide a sitemap_url, '
#                             'and the sitemap URL could not be auto-detected.')
#                 self.log_error(message)
# 
#         return sitemap_url
# 
#     def get_sitemap_links(self):
#         urls = []
#         sitemap_url = self.get_sitemap_url()
#         if sitemap_url:
#             response = self.client.get(sitemap_url)
#             if not is_success(response.status_code):  # pragma: no cover
#                 message = _('Error retrieving sitemap: {sitemap_url} - Code: {code}').format(
#                     sitemap_url=sitemap_url, code=response.status_code)
#                 self.log_error(message)
#             else:
#                 soup = BeautifulSoup(response.content, 'lxml')
#                 urls = [loc.get_text() for loc in soup.find_all('loc')]
# 
#         return urls
# 
#     @staticmethod
#     def get_links(response):
#         links = []
# 
#         if is_redirect(response.status_code):
#             links.append(response['Location'])
# 
#         if is_success(response.status_code):
#             soup = BeautifulSoup(response.content,  'lxml')
#             for link in soup.find_all('a', href=True):
#                 links.append(link['href'])
# 
#         return links
# 
#     def get_urls(self):
#         urls = []
# 
#         sitemap_links = self.get_sitemap_links()
#         for url in sitemap_links:
#             parsed_url = urlparse(url)
#             self.url_registry.enqueue(parsed_url.path)
# 
#         for url in self.url_registry:
#             urls.append(url)
# 
#             response = self.client.get(url)
#             for link in self.get_links(response):
#                 cleaned_url = self.clean_url(link)
#                 if cleaned_url:
#                     parsed_url = urlparse(cleaned_url)
#                     self.url_registry.enqueue(parsed_url.path)
# 
#         urls = list(set(urls))
#         urls.sort()
#         return urls


from django.conf import settings
from django.test import TestCase, override_settings
from staticgen.exceptions import StaticgenError
from staticgen.staticgen_crawler import StaticgenCrawler

    def test_get_sitemap_links(self):
        links = self.crawler.get_sitemap_links()
        # should be 9 urls - for 9 posts urls in sitemap
        self.assertEqual(len(links), 9)

    def test_clean_urls_with_bad_urls(self):
        bad_urls = (
            '{media_url}user-upload.jpg'.format(media_url=settings.MEDIA_URL),
            '{static_url}logo.jpg'.format(static_url=settings.STATIC_URL),
            '#page-top',
            'mailto:staticgen@example.com',
            'http://twitter.com/staticgen/'
        )

        for url in bad_urls:
            self.assertEqual(self.crawler.clean_url(url), None)

    def test_clean_urls_with_good_url(self):
        good_urls = (
            ('/staticgen/', '/staticgen/'),
            ('http://example.com/staticgen/', '//example.com/staticgen/'),
            ('https://example.com/staticgen/', '//example.com/staticgen/'),
            ('http://www.example.com/staticgen/', '//example.com/staticgen/'),
            ('https://www.example.com/staticgen/', '//example.com/staticgen/'),
        )

        for url, cleaned_url in good_urls:
            self.assertEqual(self.crawler.clean_url(url), cleaned_url)


# Path: staticgen/exceptions.py
# class StaticgenError(Exception):
#     pass
# 
# Path: staticgen/staticgen_crawler.py
# class StaticgenCrawler(object):
# 
#     def __init__(self):
#         current_site = Site.objects.get_current()
#         self.base_domain = current_site.domain
#         self.url_registry = UrlRegistry()
#         self.client = Client(SERVER_NAME=get_static_site_domain())
# 
#     def log_error(self, message):
#         logger.error(message)
#         if not settings.STATICGEN_FAIL_SILENTLY:
#             raise StaticgenError(message)
# 
#     def clean_url(self, url):
#         url = url.lower().strip()
# 
#         special_links = [
#             url.startswith(prefix) for prefix in ('mailto', 'tel', 'sms', 'skype', 'geo')]
# 
#         if any([url.startswith(settings.MEDIA_URL),  # skip media files urls
#                 url.startswith(settings.STATIC_URL),  # skip static files urls
#                 url.startswith('#')] + special_links):  # skip anchors & special links e.g 'tel:'
#             return None
# 
#         if url.startswith('/'):
#             return url
# 
#         # strip web prefixes
#         for prefix in ('http://', 'https://', 'www.', ):
#             if url.startswith(prefix):
#                 url = url[len(prefix):]
# 
#         if not url.startswith(self.base_domain):
#             return None
# 
#         return '//{url}'.format(url=url)
# 
#     def get_sitemap_url(self):
#         sitemap_url = settings.STATICGEN_SITEMAP_URL
#         if sitemap_url is None:
#             try:
#                 # Try for the "global" sitemap URL.
#                 sitemap_url = reverse('django.contrib.sitemaps.views.sitemap')
#             except NoReverseMatch:
#                 message = _('You did not provide a sitemap_url, '
#                             'and the sitemap URL could not be auto-detected.')
#                 self.log_error(message)
# 
#         return sitemap_url
# 
#     def get_sitemap_links(self):
#         urls = []
#         sitemap_url = self.get_sitemap_url()
#         if sitemap_url:
#             response = self.client.get(sitemap_url)
#             if not is_success(response.status_code):  # pragma: no cover
#                 message = _('Error retrieving sitemap: {sitemap_url} - Code: {code}').format(
#                     sitemap_url=sitemap_url, code=response.status_code)
#                 self.log_error(message)
#             else:
#                 soup = BeautifulSoup(response.content, 'lxml')
#                 urls = [loc.get_text() for loc in soup.find_all('loc')]
# 
#         return urls
# 
#     @staticmethod
#     def get_links(response):
#         links = []
# 
#         if is_redirect(response.status_code):
#             links.append(response['Location'])
# 
#         if is_success(response.status_code):
#             soup = BeautifulSoup(response.content,  'lxml')
#             for link in soup.find_all('a', href=True):
#                 links.append(link['href'])
# 
#         return links
# 
#     def get_urls(self):
#         urls = []
# 
#         sitemap_links = self.get_sitemap_links()
#         for url in sitemap_links:
#             parsed_url = urlparse(url)
#             self.url_registry.enqueue(parsed_url.path)
# 
#         for url in self.url_registry:
#             urls.append(url)
# 
#             response = self.client.get(url)
#             for link in self.get_links(response):
#                 cleaned_url = self.clean_url(link)
#                 if cleaned_url:
#                     parsed_url = urlparse(cleaned_url)
#                     self.url_registry.enqueue(parsed_url.path)
# 
#         urls = list(set(urls))
#         urls.sort()
#         return urls


# Path: tests/test_staticgen_crawler.py
from django.conf import settings
from django.test import TestCase, override_settings
from staticgen.exceptions import StaticgenError
from staticgen.staticgen_crawler import StaticgenCrawler


    def test_get_sitemap_links(self):
        links = self.crawler.get_sitemap_links()
        # should be 9 urls - for 9 posts urls in sitemap
        self.assertEqual(len(links), 9)

    def test_clean_urls_with_bad_urls(self):
        bad_urls = (
            '{media_url}user-upload.jpg'.format(media_url=settings.MEDIA_URL),
            '{static_url}logo.jpg'.format(static_url=settings.STATIC_URL),
            '#page-top',
            'mailto:staticgen@example.com',
            'http://twitter.com/staticgen/'
        )

        for url in bad_urls:
            self.assertEqual(self.crawler.clean_url(url), None)

    def test_clean_urls_with_good_url(self):
        good_urls = (
            ('/staticgen/', '/staticgen/'),
            ('http://example.com/staticgen/', '//example.com/staticgen/'),
            ('https://example.com/staticgen/', '//example.com/staticgen/'),
            ('http://www.example.com/staticgen/', '//example.com/staticgen/'),
            ('https://www.example.com/staticgen/', '//example.com/staticgen/'),
        )

        for url, cleaned_url in good_urls:
            self.assertEqual(self.crawler.clean_url(url), cleaned_url)


    def test_get_pages(self):