<commit_before>import mimetypes

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files import File
from django.core.files.storage import Storage
from django.utils.text import get_valid_filename

try:
    import cloudfiles
    from cloudfiles.errors import NoSuchObject, ResponseError
except ImportError:
    raise ImproperlyConfigured("Could not load cloudfiles dependency. See "
                               "http://www.mosso.com/cloudfiles.jsp.")

USERNAME = getattr(settings, 'CUMULUS_USERNAME')
API_KEY = getattr(settings, 'CUMULUS_API_KEY')
CONTAINER = getattr(settings, 'CUMULUS_CONTAINER')
TIMEOUT = getattr(settings, 'CUMULUS_TIMEOUT', 5)
USE_SERVICENET = getattr(settings, 'CUMULUS_USE_SERVICENET', False)
# TODO: implement TTL into cloudfiles methods
TTL = getattr(settings, 'CUMULUS_TTL', 600)


class CloudFilesStorage(Storage):
    """
    Custom storage for Mosso Cloud Files.
    """
    default_quick_listdir = True

    def __init__(self, username=None, api_key=None, container=None, timeout=None,
                 connection_kwargs=None):
        """
        Initialize the settings for the connection and container.
        """
        self.username = username or USERNAME
        self.api_key = api_key or API_KEY
        self.container_name = container or CONTAINER
        self.timeout = timeout or TIMEOUT
        self.use_servicenet = USE_SERVICENET
        self.connection_kwargs = connection_kwargs or {}


    def __getstate__(self):
        """
        Return a picklable representation of the storage.
        """
        return dict(username=self.username,
                    api_key=self.api_key,
                    container_name=self.container_name,
                    timeout=self.timeout,
                    use_servicenet=self.use_servicenet,
                    connection_kwargs=self.connection_kwargs)

    def _get_connection(self):
        if not hasattr(self, '_connection'):
            self._connection = cloudfiles.get_connection(
                                  username=self.username,
                                  api_key=self.api_key,
                                  timeout=self.timeout,
                                  servicenet=self.use_servicenet,
                                  **self.connection_kwargs)
        return self._connection

    def _set_connection(self, value):
        self._connection = value

    connection = property(_get_connection, _set_connection)

    def _get_container(self):
        if not hasattr(self, '_container'):
            self.container = self.connection.get_container(
                                                        self.container_name)
        return self._container

    def _set_container(self, container):
        """
        Set the container, making it publicly available (on Limelight CDN) if
        it is not already.
        """
        if not container.is_public():
            container.make_public()
        if hasattr(self, '_container_public_uri'):
            delattr(self, '_container_public_uri')
        self._container = container

    container = property(_get_container, _set_container)

    def _get_container_url(self):
        if not hasattr(self, '_container_public_uri'):
            self._container_public_uri = self.container.public_uri()
        return self._container_public_uri

    container_url = property(_get_container_url)

    def _get_cloud_obj(self, name):
        """
        Helper function to get retrieve the requested Cloud Files Object.
        """
        return self.container.get_object(name)

    def _open(self, name, mode='rb'):
        """
        Return the CloudFilesStorageFile.
        """
        return CloudFilesStorageFile(storage=self, name=name)

    def _save(self, name, content):
        """
        Use the Cloud Files service to write ``content`` to a remote file
        (called ``name``).
        """
        content.open()
        cloud_obj = self.container.create_object(name)<commit_msg>Added support for CNAMEs via new setting CUMULUS_CNAMES (dictionary). If setting is defined and the container URL is defined therein, the container URL will be set to its value.<commit_after>import mimetypes

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files import File
from django.core.files.storage import Storage
from django.utils.text import get_valid_filename

try:
    import cloudfiles
    from cloudfiles.errors import NoSuchObject, ResponseError
except ImportError:
    raise ImproperlyConfigured("Could not load cloudfiles dependency. See "
                               "http://www.mosso.com/cloudfiles.jsp.")

USERNAME = getattr(settings, 'CUMULUS_USERNAME')
API_KEY = getattr(settings, 'CUMULUS_API_KEY')
CONTAINER = getattr(settings, 'CUMULUS_CONTAINER')
TIMEOUT = getattr(settings, 'CUMULUS_TIMEOUT', 5)
USE_SERVICENET = getattr(settings, 'CUMULUS_USE_SERVICENET', False)
CNAMES = getattr(settings, 'CUMULUS_CNAMES', None)
# TODO: implement TTL into cloudfiles methods
TTL = getattr(settings, 'CUMULUS_TTL', 600)


class CloudFilesStorage(Storage):
    """
    Custom storage for Mosso Cloud Files.
    """
    default_quick_listdir = True

    def __init__(self, username=None, api_key=None, container=None, timeout=None,
                 connection_kwargs=None):
        """
        Initialize the settings for the connection and container.
        """
        self.username = username or USERNAME
        self.api_key = api_key or API_KEY
        self.container_name = container or CONTAINER
        self.timeout = timeout or TIMEOUT
        self.use_servicenet = USE_SERVICENET
        self.connection_kwargs = connection_kwargs or {}


    def __getstate__(self):
        """
        Return a picklable representation of the storage.
        """
        return dict(username=self.username,
                    api_key=self.api_key,
                    container_name=self.container_name,
                    timeout=self.timeout,
                    use_servicenet=self.use_servicenet,
                    connection_kwargs=self.connection_kwargs)

    def _get_connection(self):
        if not hasattr(self, '_connection'):
            self._connection = cloudfiles.get_connection(
                                  username=self.username,
                                  api_key=self.api_key,
                                  timeout=self.timeout,
                                  servicenet=self.use_servicenet,
                                  **self.connection_kwargs)
        return self._connection

    def _set_connection(self, value):
        self._connection = value

    connection = property(_get_connection, _set_connection)

    def _get_container(self):
        if not hasattr(self, '_container'):
            self.container = self.connection.get_container(
                                                        self.container_name)
        return self._container

    def _set_container(self, container):
        """
        Set the container, making it publicly available (on Limelight CDN) if
        it is not already.
        """
        if not container.is_public():
            container.make_public()
        if hasattr(self, '_container_public_uri'):
            delattr(self, '_container_public_uri')
        self._container = container

    container = property(_get_container, _set_container)

    def _get_container_url(self):
        if not hasattr(self, '_container_public_uri'):
            self._container_public_uri = self.container.public_uri()
        if CNAMES and self._container_public_uri in CNAMES:
            self._container_public_uri = CNAMES[self._container_public_uri]
        return self._container_public_uri

    container_url = property(_get_container_url)

    def _get_cloud_obj(self, name):
        """
        Helper function to get retrieve the requested Cloud Files Object.
        """
        return self.container.get_object(name)

    def _open(self, name, mode='rb'):
        """
        Return the CloudFilesStorageFile.
        """
        return CloudFilesStorageFile(storage=self, name=name)

    def _save(self, name, content):
        """
        Use the Cloud Files service to write ``content`` to a remote file
        (called ``name``).
        """
        content.open()
        cloud_obj = self.container.create_object(name)
{'fraction_non_alphanumeric': 0.0796628279710317, 'fraction_numerical': 0.000949780363290989, 'mean_word_length': 2.3696, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6229227', 'n_tokens_mistral': 2282, 'n_tokens_neox': 2212, 'n_words': 545}