<commit_before>import os

import mimetypes

from django.conf import settings
from django.core.files.storage import Storage
from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
from django.utils.encoding import force_unicode, smart_str

from .format import SubdomainCallingFormat
from .file import GSBotoStorageFile
from .utils import safe_join

try:
    from boto.gs.connection import GSConnection
    from boto.gs.key import Key
except ImportError:
    raise ImproperlyConfigured("Could not load Google Storage bindings.\n"
                               "See http://code.google.com/p/boto/")


ACCESS_KEY_NAME     = getattr(settings, 'GS_ACCESS_KEY_ID', None)
SECRET_KEY_NAME     = getattr(settings, 'GS_SECRET_ACCESS_KEY', None)
HEADERS             = getattr(settings, 'GS_HEADERS', {})
STORAGE_BUCKET_NAME = getattr(settings, 'GS_STORAGE_BUCKET_NAME', None)
AUTO_CREATE_BUCKET  = getattr(settings, 'GS_AUTO_CREATE_BUCKET', False)
DEFAULT_ACL         = getattr(settings, 'GS_DEFAULT_ACL', 'public-read')
BUCKET_ACL          = getattr(settings, 'GS_BUCKET_ACL', DEFAULT_ACL)
QUERYSTRING_AUTH    = getattr(settings, 'GS_QUERYSTRING_AUTH', True)
QUERYSTRING_EXPIRE  = getattr(settings, 'GS_QUERYSTRING_EXPIRE', 3600)
REDUCED_REDUNDANCY  = getattr(settings, 'GS_REDUCED_REDUNDANCY', False)
LOCATION            = getattr(settings, 'GS_LOCATION', '')
CUSTOM_DOMAIN       = getattr(settings, 'GS_CUSTOM_DOMAIN', None)
CALLING_FORMAT      = getattr(settings, 'GS_CALLING_FORMAT', SubdomainCallingFormat())
SECURE_URLS         = getattr(settings, 'GS_SECURE_URLS', True)
FILE_NAME_CHARSET   = getattr(settings, 'GS_FILE_NAME_CHARSET', 'utf-8')
FILE_OVERWRITE      = getattr(settings, 'GS_FILE_OVERWRITE', True)
IS_GZIPPED          = getattr(settings, 'GS_IS_GZIPPED', False)
PRELOAD_METADATA    = getattr(settings, 'GS_PRELOAD_METADATA', False)
GZIP_CONTENT_TYPES  = getattr(settings, 'GZIP_CONTENT_TYPES', (
    'text/css',
    'application/javascript',
    'application/x-javascript'
))


class GoogleStorage(Storage):
    def __init__(self, bucket=STORAGE_BUCKET_NAME, access_key=None,
                       secret_key=None, bucket_acl=BUCKET_ACL, acl=DEFAULT_ACL, headers=HEADERS,
                       gzip=IS_GZIPPED, gzip_content_types=GZIP_CONTENT_TYPES,
                       querystring_auth=QUERYSTRING_AUTH, querystring_expire=QUERYSTRING_EXPIRE,
                       reduced_redundancy=REDUCED_REDUNDANCY,
                       custom_domain=CUSTOM_DOMAIN, secure_urls=SECURE_URLS,
                       location=LOCATION, file_name_charset=FILE_NAME_CHARSET,
                       preload_metadata=PRELOAD_METADATA, calling_format=CALLING_FORMAT):

        self.bucket_acl = bucket_acl
        self.bucket_name = bucket
        self.acl = acl
        self.headers = headers
        self.preload_metadata = preload_metadata
        self.gzip = gzip
        self.gzip_content_types = gzip_content_types
        self.querystring_auth = querystring_auth
        self.querystring_expire = querystring_expire
        self.reduced_redundancy = reduced_redundancy
        self.custom_domain = custom_domain
        self.secure_urls = secure_urls
        self.location = location or ''
        self.location = self.location.lstrip('/')
        self.file_name_charset = file_name_charset

        if not access_key and not secret_key:
            print u'where are no secret keys.'
            access_key, secret_key = self._get_access_keys()

        self.connection = GSConnection(access_key, secret_key)

    @property
    def bucket(self):
        if not hasattr(self, '_bucket'):
            self._bucket = self._get_or_create_bucket(self.bucket_name)
        return self._bucket

    @property
    def entries(self):
        if self.preload_metadata and not self._entries:
            self._entries = dict((self._decode_name(entry.key), entry)
                                for entry in self.bucket.list())
        return self._entries

    def _get_access_keys(self):
        access_key = ACCESS_KEY_NAME
        secret_key = SECRET_KEY_NAME
        if (access_key or secret_key) and (not access_key or not secret_key):
            access_key = os.environ.get(ACCESS_KEY_NAME)
            secret_key = os.environ.get(SECRET_KEY_NAME)

        if access_key and secret_key:
            # Both were provided, so use them
            return access_key, secret_key

        return None, None<commit_msg>Fix self._entries error<commit_after>import os

import mimetypes

from django.conf import settings
from django.core.files.storage import Storage
from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
from django.utils.encoding import force_unicode, smart_str

from .format import SubdomainCallingFormat
from .file import GSBotoStorageFile
from .utils import safe_join

try:
    from boto.gs.connection import GSConnection
    from boto.gs.key import Key
except ImportError:
    raise ImproperlyConfigured("Could not load Google Storage bindings.\n"
                               "See http://code.google.com/p/boto/")


ACCESS_KEY_NAME     = getattr(settings, 'GS_ACCESS_KEY_ID', None)
SECRET_KEY_NAME     = getattr(settings, 'GS_SECRET_ACCESS_KEY', None)
HEADERS             = getattr(settings, 'GS_HEADERS', {})
STORAGE_BUCKET_NAME = getattr(settings, 'GS_STORAGE_BUCKET_NAME', None)
AUTO_CREATE_BUCKET  = getattr(settings, 'GS_AUTO_CREATE_BUCKET', False)
DEFAULT_ACL         = getattr(settings, 'GS_DEFAULT_ACL', 'public-read')
BUCKET_ACL          = getattr(settings, 'GS_BUCKET_ACL', DEFAULT_ACL)
QUERYSTRING_AUTH    = getattr(settings, 'GS_QUERYSTRING_AUTH', True)
QUERYSTRING_EXPIRE  = getattr(settings, 'GS_QUERYSTRING_EXPIRE', 3600)
REDUCED_REDUNDANCY  = getattr(settings, 'GS_REDUCED_REDUNDANCY', False)
LOCATION            = getattr(settings, 'GS_LOCATION', '')
CUSTOM_DOMAIN       = getattr(settings, 'GS_CUSTOM_DOMAIN', None)
CALLING_FORMAT      = getattr(settings, 'GS_CALLING_FORMAT', SubdomainCallingFormat())
SECURE_URLS         = getattr(settings, 'GS_SECURE_URLS', True)
FILE_NAME_CHARSET   = getattr(settings, 'GS_FILE_NAME_CHARSET', 'utf-8')
FILE_OVERWRITE      = getattr(settings, 'GS_FILE_OVERWRITE', True)
IS_GZIPPED          = getattr(settings, 'GS_IS_GZIPPED', False)
PRELOAD_METADATA    = getattr(settings, 'GS_PRELOAD_METADATA', False)
GZIP_CONTENT_TYPES  = getattr(settings, 'GZIP_CONTENT_TYPES', (
    'text/css',
    'application/javascript',
    'application/x-javascript'
))


class GoogleStorage(Storage):
    def __init__(self, bucket=STORAGE_BUCKET_NAME, access_key=None,
                       secret_key=None, bucket_acl=BUCKET_ACL, acl=DEFAULT_ACL, headers=HEADERS,
                       gzip=IS_GZIPPED, gzip_content_types=GZIP_CONTENT_TYPES,
                       querystring_auth=QUERYSTRING_AUTH, querystring_expire=QUERYSTRING_EXPIRE,
                       reduced_redundancy=REDUCED_REDUNDANCY,
                       custom_domain=CUSTOM_DOMAIN, secure_urls=SECURE_URLS,
                       location=LOCATION, file_name_charset=FILE_NAME_CHARSET,
                       preload_metadata=PRELOAD_METADATA, calling_format=CALLING_FORMAT):

        self.bucket_acl = bucket_acl
        self.bucket_name = bucket
        self.acl = acl
        self.headers = headers
        self.preload_metadata = preload_metadata
        self.gzip = gzip
        self.gzip_content_types = gzip_content_types
        self.querystring_auth = querystring_auth
        self.querystring_expire = querystring_expire
        self.reduced_redundancy = reduced_redundancy
        self.custom_domain = custom_domain
        self.secure_urls = secure_urls
        self.location = location or ''
        self.location = self.location.lstrip('/')
        self.file_name_charset = file_name_charset

        if not access_key and not secret_key:
            print u'where are no secret keys.'
            access_key, secret_key = self._get_access_keys()

        self.connection = GSConnection(access_key, secret_key)
        
        self._entries = {}

    @property
    def bucket(self):
        if not hasattr(self, '_bucket'):
            self._bucket = self._get_or_create_bucket(self.bucket_name)
        return self._bucket

    @property
    def entries(self):
        if self.preload_metadata and not self._entries:
            self._entries = dict((self._decode_name(entry.key), entry)
                                for entry in self.bucket.list())
        return self._entries

    def _get_access_keys(self):
        access_key = ACCESS_KEY_NAME
        secret_key = SECRET_KEY_NAME
        if (access_key or secret_key) and (not access_key or not secret_key):
            access_key = os.environ.get(ACCESS_KEY_NAME)
            secret_key = os.environ.get(SECRET_KEY_NAME)

        if access_key and secret_key:
            # Both were provided, so use them
            return access_key, secret_key

        return None, None
{'fraction_non_alphanumeric': 0.08013898229096615, 'fraction_numerical': 0.0011208249271463797, 'mean_word_length': 3.1483031148303113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7566739', 'n_tokens_mistral': 2840, 'n_tokens_neox': 2791, 'n_words': 530}