<commit_before>import mimetypes
import os
import pyrax
import StringIO
import swiftclient

from django.conf import settings
from django.core.files import File
from django.core.files.storage import Storage
from django.core.management import call_command

from cumulus.cloudfiles_cdn import CloudfilesCDN
from cumulus.settings import CUMULUS


class SwiftclientStorage(Storage):
    """
    Custom storage for Swiftclient.
    """
    default_quick_listdir = True

    def __init__(self, username=None, api_key=None, container=None,
                 connection_kwargs=None):
        """
        Initialize the settings for the connection and container.
        """
        api_key = api_key or CUMULUS["API_KEY"]
        username = username or CUMULUS["USERNAME"]
        pyrax.set_credentials(username, api_key)
        # self.auth_url = CUMULUS["AUTH_URL"]
        # self.connection_kwargs = connection_kwargs or {}
        self.container_name = container or CUMULUS["CONTAINER"]
        self.use_snet = CUMULUS["SERVICENET"]
        self.use_ssl = CUMULUS["USE_SSL"]
        self.container = self.get_or_create_container()
        self.container_public_uri = self.get_container_uri()

    def __getstate__(self):
        """
        Return a picklable representation of the storage.
        """
        return {
            "username": self.username,
            "api_key": self.api_key,
            "container_name": self.container_name,
            "use_snet": self.use_snet,
            "connection_kwargs": self.connection_kwargs
        }

    def get_or_create_container(self):
        """
        Get the container, creating and/or making it publicly available if
        it is not already.
        """
        container = pyrax.cloudfiles.create_container(self.container_name)
        if not container.cdn_enabled:
            container.make_public()
        return container

    def get_container_uri(self):
        if self.use_ssl and CUMULUS["CONTAINER_SSL_URI"]:
            container_public_uri = CUMULUS["CONTAINER_SSL_URI"]
        elif self.use_ssl:
            container_public_uri = self.container.cdn_ssl_uri
        elif CUMULUS["CONTAINER_URI"]:
            container_public_uri = CUMULUS["CONTAINER_URI"]
        else:
            container_public_uri = self.container.cdn_uri
        if CUMULUS["CNAMES"] and container_public_uri in CUMULUS["CNAMES"]:
            container_public_uri = CUMULUS["CNAMES"][container_public_uri]
        return container_public_uri

    def _get_cloud_obj(self, name):
        """
        Helper function to retrieve the requested Cloud Files Object.
        """
        if name not in self.container.get_object_names():
            return False
        else:
            return self.container.get_object(name)

    def _open(self, name, mode="rb"):
        """
        Return the SwiftclientStorageFile.
        """
        return SwiftclientStorageFile(storage=self, name=name)

    def _save(self, name, content):
        """
        Use the Swiftclient service to write ``content`` to a remote
        file (called ``name``).
        """
        # Checks if the content_type is already set.
        # Otherwise uses the mimetypes library to guess.
        if hasattr(content.file, "content_type"):
            content_type = content.file.content_type
        else:
            mime_type, encoding = mimetypes.guess_type(name)
            content_type = mime_type
        if name not in self.container.get_object_names():
            self.container.store_object(obj_name=name,
                                        data=content.read(),
                                        content_type=content_type,
                                        etag=None)
        return name

    def delete(self, name):
        """
        Deletes the specified file from the storage system.

        Deleting a model doesn't delete associated files:
        https://docs.djangoproject.com/en/1.3/releases/1.3/#deleting-a-model-doesn-t-delete-associated-files
        """
        try:
            self.container.delete_object(name)
        except swiftclient.client.ClientException, exc:
            if exc.http_status == 404:
                pass
            else:
                raise

    def exists(self, name):
        """
        Returns True if a file referenced by the given name already
        exists in the storage system, or False if the name is
        available for a new file.<commit_msg>cleanup imports<commit_after>import mimetypes
import pyrax

from django.conf import settings
from django.core.files import File
from django.core.files.storage import Storage

from cumulus.settings import CUMULUS


class SwiftclientStorage(Storage):
    """
    Custom storage for Swiftclient.
    """
    default_quick_listdir = True

    def __init__(self, username=None, api_key=None, container=None,
                 connection_kwargs=None):
        """
        Initialize the settings for the connection and container.
        """
        api_key = api_key or CUMULUS["API_KEY"]
        username = username or CUMULUS["USERNAME"]
        pyrax.set_credentials(username, api_key)
        # self.auth_url = CUMULUS["AUTH_URL"]
        # self.connection_kwargs = connection_kwargs or {}
        self.container_name = container or CUMULUS["CONTAINER"]
        self.use_snet = CUMULUS["SERVICENET"]
        self.use_ssl = CUMULUS["USE_SSL"]
        self.container = self.get_or_create_container()
        self.container_public_uri = self.get_container_uri()

    def __getstate__(self):
        """
        Return a picklable representation of the storage.
        """
        return {
            "username": self.username,
            "api_key": self.api_key,
            "container_name": self.container_name,
            "use_snet": self.use_snet,
            "connection_kwargs": self.connection_kwargs
        }

    def get_or_create_container(self):
        """
        Get the container, creating and/or making it publicly available if
        it is not already.
        """
        container = pyrax.cloudfiles.create_container(self.container_name)
        if not container.cdn_enabled:
            container.make_public()
        return container

    def get_container_uri(self):
        if self.use_ssl and CUMULUS["CONTAINER_SSL_URI"]:
            container_public_uri = CUMULUS["CONTAINER_SSL_URI"]
        elif self.use_ssl:
            container_public_uri = self.container.cdn_ssl_uri
        elif CUMULUS["CONTAINER_URI"]:
            container_public_uri = CUMULUS["CONTAINER_URI"]
        else:
            container_public_uri = self.container.cdn_uri
        if CUMULUS["CNAMES"] and container_public_uri in CUMULUS["CNAMES"]:
            container_public_uri = CUMULUS["CNAMES"][container_public_uri]
        return container_public_uri

    def _get_cloud_obj(self, name):
        """
        Helper function to retrieve the requested Cloud Files Object.
        """
        if name not in self.container.get_object_names():
            return False
        else:
            return self.container.get_object(name)

    def _open(self, name, mode="rb"):
        """
        Return the SwiftclientStorageFile.
        """
        return SwiftclientStorageFile(storage=self, name=name)

    def _save(self, name, content):
        """
        Use the Swiftclient service to write ``content`` to a remote
        file (called ``name``).
        """
        # Checks if the content_type is already set.
        # Otherwise uses the mimetypes library to guess.
        if hasattr(content.file, "content_type"):
            content_type = content.file.content_type
        else:
            mime_type, encoding = mimetypes.guess_type(name)
            content_type = mime_type
        if name not in self.container.get_object_names():
            self.container.store_object(obj_name=name,
                                        data=content.read(),
                                        content_type=content_type,
                                        etag=None)
        return name

    def delete(self, name):
        """
        Deletes the specified file from the storage system.

        Deleting a model doesn't delete associated files:
        https://docs.djangoproject.com/en/1.3/releases/1.3/#deleting-a-model-doesn-t-delete-associated-files
        """
        try:
            self.container.delete_object(name)
        except pyrax.exceptions.ClientException, exc:
            if exc.http_status == 404:
                pass
            else:
                raise

    def exists(self, name):
        """
        Returns True if a file referenced by the given name already
        exists in the storage system, or False if the name is
        available for a new file.
{'fraction_non_alphanumeric': 0.08259081562714188, 'fraction_numerical': 0.0015992689056431345, 'mean_word_length': 2.164076617275027, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1156322', 'n_tokens_mistral': 2450, 'n_tokens_neox': 2365, 'n_words': 588}