<commit_before><commit_msg>storage<commit_after>from boto.utils import parse_ts
from django.core.files.base import File
from django.core.files.storage import get_storage_class
from storages.backends.s3boto import S3BotoStorage

import os

# From https://github.com/jezdez/django_compressor/issues/100

class CachedS3BotoStorage(S3BotoStorage):
    def __init__(self, *args, **kwargs):
        super(CachedS3BotoStorage, self).__init__(*args, **kwargs)
        self.local_storage = get_storage_class('django.core.files.storage.FileSystemStorage')()

    def _open(self, name, mode='rb'):
        original_file = super(CachedS3BotoStorage, self)._open(name, mode=mode)
        if name.endswith('.gz'):
            return original_file
        return original_file

    def modified_time(self, name):
        name = self._normalize_name(self._clean_name(name))
        entry = self.entries.get(name)
        if entry is None:
            entry = self.bucket.get_key(self._encode_name(name))
        # Parse the last_modified string to a local datetime object.
        return parse_ts(entry.last_modified)

    def path(self, name):
        return self.local_storage.path(name)
{'fraction_non_alphanumeric': 0.10144927536231885, 'fraction_numerical': 0.0076726342710997444, 'mean_word_length': 3.6772908366533863, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1971460', 'n_tokens_mistral': 365, 'n_tokens_neox': 357, 'n_words': 74}