<commit_before>"""
Qiniu Storage Backends
"""
from __future__ import absolute_import
import datetime
import os
from urlparse import urljoin

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

import qiniu.conf
import qiniu.io
import qiniu.rs
import qiniu.rsf
import requests

from django.conf import settings
from django.core.files.base import File
from django.core.files.storage import Storage
from django.core.exceptions import ImproperlyConfigured

from .utils import bucket_lister


def get_qiniu_config(name):
    """
    Get configuration variable from environment variable
    or django setting.py
    """
    config = os.environ.get(name, getattr(settings, name, None))
    if config:
        return config
    else:
        raise ImproperlyConfigured(
            "Can't find config for '%s' either in environment"
            "variable or in setting.py" % name)


QINIU_ACCESS_KEY = get_qiniu_config('QINIU_ACCESS_KEY')
QINIU_SECRET_KEY = get_qiniu_config('QINIU_SECRET_KEY')
QINIU_BUCKET_NAME = get_qiniu_config('QINIU_BUCKET_NAME')
QINIU_BUCKET_DOMAIN = get_qiniu_config('QINIU_BUCKET_DOMAIN')


class QiniuStorage(Storage):
    """
    Qiniu Storage Service
    """
    location = ""
    def __init__(
            self,
            access_key=QINIU_ACCESS_KEY,
            secret_key=QINIU_SECRET_KEY,
            bucket_name=QINIU_BUCKET_NAME,
            bucket_domain=QINIU_BUCKET_DOMAIN):
        qiniu.conf.ACCESS_KEY = access_key
        qiniu.conf.SECRET_KEY = secret_key
        self.bucket_name = bucket_name
        self.put_policy = qiniu.rs.PutPolicy(self.bucket_name)
        self.bucket_domain = bucket_domain

    def _clean_name(self, name):
        if type(name) is unicode:
            return name.encode('utf-8')
        else:
            return name
    def _normalize_name(self, name):
        return ("%s/%s"% (self.location, name.lstrip('/'))).lstrip('/')

    def _open(self, name, mode='rb'):
        return QiniuFile(name, self, mode)

    def _save(self, name, content):
        name = self._normalize_name(self._clean_name(name))

        if hasattr(content, 'open'):
            # Since Django 1.6, content should be a instance
            # of `django.core.files.File`
            content.open()

        if hasattr(content, 'chunks'):
            content_str = ''.join(chunk for chunk in content.chunks())
        else:
            content_str = content.read()

        self._put_file(name, content_str)
        content.close()
        return name

    def _put_file(self, name, content):
        token = self.put_policy.token()
        print token
        ret, err = qiniu.io.put(token, name, content)
        if err:
            raise IOError(
                "Failed to put file '%s'. "
                "Error message: %s" % (name, err))

    def _read(self, name):
        return requests.get(self.url(name)).content

    def delete(self, name):
        name = self._normalize_name(self._clean_name(name))
        ret, err = qiniu.rs.Client().delete(self.bucket_name, name)
        if err:
            raise IOError(
                "Failed to delete file '%s'. "
                "Error message: %s" % (name, err))

    def _file_stat(self, name, silent=False):<commit_msg>Remove debug info<commit_after>"""
Qiniu Storage Backends
"""
from __future__ import absolute_import
import datetime
import os
from urlparse import urljoin

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

import qiniu.conf
import qiniu.io
import qiniu.rs
import qiniu.rsf
import requests

from django.conf import settings
from django.core.files.base import File
from django.core.files.storage import Storage
from django.core.exceptions import ImproperlyConfigured

from .utils import bucket_lister


def get_qiniu_config(name):
    """
    Get configuration variable from environment variable
    or django setting.py
    """
    config = os.environ.get(name, getattr(settings, name, None))
    if config:
        return config
    else:
        raise ImproperlyConfigured(
            "Can't find config for '%s' either in environment"
            "variable or in setting.py" % name)


QINIU_ACCESS_KEY = get_qiniu_config('QINIU_ACCESS_KEY')
QINIU_SECRET_KEY = get_qiniu_config('QINIU_SECRET_KEY')
QINIU_BUCKET_NAME = get_qiniu_config('QINIU_BUCKET_NAME')
QINIU_BUCKET_DOMAIN = get_qiniu_config('QINIU_BUCKET_DOMAIN')


class QiniuStorage(Storage):
    """
    Qiniu Storage Service
    """
    location = ""
    def __init__(
            self,
            access_key=QINIU_ACCESS_KEY,
            secret_key=QINIU_SECRET_KEY,
            bucket_name=QINIU_BUCKET_NAME,
            bucket_domain=QINIU_BUCKET_DOMAIN):
        qiniu.conf.ACCESS_KEY = access_key
        qiniu.conf.SECRET_KEY = secret_key
        self.bucket_name = bucket_name
        self.put_policy = qiniu.rs.PutPolicy(self.bucket_name)
        self.bucket_domain = bucket_domain

    def _clean_name(self, name):
        if type(name) is unicode:
            return name.encode('utf-8')
        else:
            return name
    def _normalize_name(self, name):
        return ("%s/%s"% (self.location, name.lstrip('/'))).lstrip('/')

    def _open(self, name, mode='rb'):
        return QiniuFile(name, self, mode)

    def _save(self, name, content):
        name = self._normalize_name(self._clean_name(name))

        if hasattr(content, 'open'):
            # Since Django 1.6, content should be a instance
            # of `django.core.files.File`
            content.open()

        if hasattr(content, 'chunks'):
            content_str = ''.join(chunk for chunk in content.chunks())
        else:
            content_str = content.read()

        self._put_file(name, content_str)
        content.close()
        return name

    def _put_file(self, name, content):
        token = self.put_policy.token()
        ret, err = qiniu.io.put(token, name, content)
        if err:
            raise IOError(
                "Failed to put file '%s'. "
                "Error message: %s" % (name, err))

    def _read(self, name):
        return requests.get(self.url(name)).content

    def delete(self, name):
        name = self._normalize_name(self._clean_name(name))
        ret, err = qiniu.rs.Client().delete(self.bucket_name, name)
        if err:
            raise IOError(
                "Failed to delete file '%s'. "
                "Error message: %s" % (name, err))

    def _file_stat(self, name, silent=False):
{'fraction_non_alphanumeric': 0.0985569542523795, 'fraction_numerical': 0.00092109303039607, 'mean_word_length': 2.5855806274078152, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1518152', 'n_tokens_mistral': 2089, 'n_tokens_neox': 2038, 'n_words': 493}