# Path: staticgen/conf.py
# class StaticgenConf(AppConf):
#     class Meta:
#     SITEMAP_URL = None
#     FAIL_SILENTLY = True
#     MULTITHREAD = True
#     BUCKET_NAME = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', '')
#     STATIC_SITE_DOMAIN = None
# 
# Path: staticgen/helpers.py
# def get_static_site_domain():
#     static_site_domain = settings.STATICGEN_STATIC_SITE_DOMAIN
#     if static_site_domain is None:
#         connection = connect_s3(
#             settings.AWS_ACCESS_KEY_ID,
#             settings.AWS_SECRET_ACCESS_KEY
#         )
#         bucket = connection.get_bucket(settings.AWS_STORAGE_BUCKET_NAME)
#         static_site_domain = bucket.get_website_endpoint()
#     return static_site_domain.rstrip('/')
# 
# Path: staticgen/models.py
# class LogEntry(models.Model):
#     site = models.ForeignKey(Site, verbose_name=_('Site'))
#     page = models.ForeignKey(Page, blank=True, null=True, on_delete=models.CASCADE)
#     change_message = models.TextField(_('change message'))
#     action_time = models.DateTimeField(_('action time'), auto_now=True)
# 
#     objects = LogEntryManager()
# 
#     class Meta:
#         verbose_name = _('log entry')
#         verbose_name_plural = _('log entries')
#         ordering = ('-action_time',)
# 
#     def __str__(self):
#         return self.change_message
# 
# class Page(models.Model):
#     PUBLISHER_STATE_DEFAULT = 0
#     PUBLISHER_STATE_CHANGED = 1
#     PUBLISHER_STATE_PUBLISHED = 2
#     PUBLISHER_STATE_ERROR = 3
#     PUBLISHER_STATE_DELETE = 4
# 
#     PUBLISHER_STATES = (
#         (PUBLISHER_STATE_DEFAULT, _('Pending')),
#         (PUBLISHER_STATE_CHANGED, _('Changed')),
#         (PUBLISHER_STATE_PUBLISHED, _('Published')),
#         (PUBLISHER_STATE_ERROR, _('Error')),
#         (PUBLISHER_STATE_DELETE, _('Deleted'))
#     )
# 
#     site = models.ForeignKey(Site, verbose_name=_('Site'))
#     path = models.CharField(_('URL'), max_length=255)
#     publisher_state = models.SmallIntegerField(
#         _('State'), choices=PUBLISHER_STATES, default=PUBLISHER_STATE_DEFAULT,
#         editable=False, db_index=True)
#     updated_at = models.DateTimeField(auto_now=True)
#     objects = PageManager()
# 
#     class Meta:
#         verbose_name = _('page')
#         verbose_name_plural = _('pages')
#         ordering = ('path', )
#         unique_together = ('site', 'path')
# 
#     def __str__(self):
#         return self.path
# 
#     @property
#     def has_changed(self):
#         return self.publisher_state == self.PUBLISHER_STATE_CHANGED
# 
# Path: staticgen/signals.py



import logging
import os
import time
from multiprocessing.pool import ThreadPool
from django.core.files.base import ContentFile
from django.test import Client
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _
from boto import connect_s3
from .conf import settings
from .helpers import get_static_site_domain
from .models import LogEntry, Page
from .signals import publishing_complete
from .status import is_redirect, is_success
    from urllib.parse import urlparse
    from urlparse import urlparse
            for chunk in key_chunks:
                bucket.delete_keys(chunk)

            message = _('Successfully deleted {deleted_count} pages.').format(
                deleted_count=len(to_delete))
            logger.info(message)

            # finally delete records from the database
            deleted_pages.delete()

    def pre_publish(self, sync_pages=True):
        # start timer
        self.start_time = time.time()

    def post_publish(self):
        # delete pages that have been marked for deletion
        self.delete_removed()

        elapsed_time = time.time() - self.start_time
        message = _('Publishing completed successfully. {updated_count} page(s) was updated, '
                    'deleted {deleted_count} page(s), in {elapsed_time:.2f} seconds.').format(
            updated_count=len(self.updated_paths),  deleted_count=len(self.deleted_paths),
            elapsed_time=elapsed_time)

        publishing_complete.send(
            sender=self.__class__,
            updated_paths=self.updated_paths,
            deleted_paths=self.deleted_paths
        )


# Path: staticgen/conf.py
# class StaticgenConf(AppConf):
#     class Meta:
#     SITEMAP_URL = None
#     FAIL_SILENTLY = True
#     MULTITHREAD = True
#     BUCKET_NAME = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', '')
#     STATIC_SITE_DOMAIN = None
# 
# Path: staticgen/helpers.py
# def get_static_site_domain():
#     static_site_domain = settings.STATICGEN_STATIC_SITE_DOMAIN
#     if static_site_domain is None:
#         connection = connect_s3(
#             settings.AWS_ACCESS_KEY_ID,
#             settings.AWS_SECRET_ACCESS_KEY
#         )
#         bucket = connection.get_bucket(settings.AWS_STORAGE_BUCKET_NAME)
#         static_site_domain = bucket.get_website_endpoint()
#     return static_site_domain.rstrip('/')
# 
# Path: staticgen/models.py
# class LogEntry(models.Model):
#     site = models.ForeignKey(Site, verbose_name=_('Site'))
#     page = models.ForeignKey(Page, blank=True, null=True, on_delete=models.CASCADE)
#     change_message = models.TextField(_('change message'))
#     action_time = models.DateTimeField(_('action time'), auto_now=True)
# 
#     objects = LogEntryManager()
# 
#     class Meta:
#         verbose_name = _('log entry')
#         verbose_name_plural = _('log entries')
#         ordering = ('-action_time',)
# 
#     def __str__(self):
#         return self.change_message
# 
# class Page(models.Model):
#     PUBLISHER_STATE_DEFAULT = 0
#     PUBLISHER_STATE_CHANGED = 1
#     PUBLISHER_STATE_PUBLISHED = 2
#     PUBLISHER_STATE_ERROR = 3
#     PUBLISHER_STATE_DELETE = 4
# 
#     PUBLISHER_STATES = (
#         (PUBLISHER_STATE_DEFAULT, _('Pending')),
#         (PUBLISHER_STATE_CHANGED, _('Changed')),
#         (PUBLISHER_STATE_PUBLISHED, _('Published')),
#         (PUBLISHER_STATE_ERROR, _('Error')),
#         (PUBLISHER_STATE_DELETE, _('Deleted'))
#     )
# 
#     site = models.ForeignKey(Site, verbose_name=_('Site'))
#     path = models.CharField(_('URL'), max_length=255)
#     publisher_state = models.SmallIntegerField(
#         _('State'), choices=PUBLISHER_STATES, default=PUBLISHER_STATE_DEFAULT,
#         editable=False, db_index=True)
#     updated_at = models.DateTimeField(auto_now=True)
#     objects = PageManager()
# 
#     class Meta:
#         verbose_name = _('page')
#         verbose_name_plural = _('pages')
#         ordering = ('path', )
#         unique_together = ('site', 'path')
# 
#     def __str__(self):
#         return self.path
# 
#     @property
#     def has_changed(self):
#         return self.publisher_state == self.PUBLISHER_STATE_CHANGED
# 
# Path: staticgen/signals.py



# Path: staticgen/publisher.py
import logging
import os
import time
from multiprocessing.pool import ThreadPool
from django.core.files.base import ContentFile
from django.test import Client
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _
from boto import connect_s3
from .conf import settings
from .helpers import get_static_site_domain
from .models import LogEntry, Page
from .signals import publishing_complete
from .status import is_redirect, is_success
    from urllib.parse import urlparse
    from urlparse import urlparse

            for chunk in key_chunks:
                bucket.delete_keys(chunk)

            message = _('Successfully deleted {deleted_count} pages.').format(
                deleted_count=len(to_delete))
            logger.info(message)

            # finally delete records from the database
            deleted_pages.delete()

    def pre_publish(self, sync_pages=True):
        # start timer
        self.start_time = time.time()

    def post_publish(self):
        # delete pages that have been marked for deletion
        self.delete_removed()

        elapsed_time = time.time() - self.start_time
        message = _('Publishing completed successfully. {updated_count} page(s) was updated, '
                    'deleted {deleted_count} page(s), in {elapsed_time:.2f} seconds.').format(
            updated_count=len(self.updated_paths),  deleted_count=len(self.deleted_paths),
            elapsed_time=elapsed_time)

        publishing_complete.send(
            sender=self.__class__,
            updated_paths=self.updated_paths,
            deleted_paths=self.deleted_paths
        )


        LogEntry.objects.log_action(message)