<commit_before>from django.core.cache import cache
from django.http import HttpResponse

class DownloadBase(object):
    """
    A basic download object.
    """
    
    def __init__(self, mimetype="text/plain", 
                 content_disposition="attachment; filename=download.txt", 
                 transfer_encoding=None, extras={}):
        self.mimetype = mimetype
        self.content_disposition = content_disposition
        self.transfer_encoding = transfer_encoding
        self.extras = {}
        
    def get_content(self):
        raise NotImplemented("Use CachedDownload or FileDownload!")
    
    def toHttpResponse(self):
        response = HttpResponse(self.get_content(), mimetype=self.mimetype)
        if self.transfer_encoding is not None:
            response['Transfer-Encoding'] = self.transfer_encoding
        response['Content-Disposition'] = self.content_disposition
        for k,v in self.extras.items():
            response[k] = v
        return response
    
    def __str__(self):
        return "content-type: %s, disposition: %s" % (self.mimetype, self.content_disposition)

class CachedDownload(DownloadBase):
    """
    Download that lives in the cache
    """
    
    def __init__(self, cacheindex, mimetype="text/plain", 
                 content_disposition="attachment; filename=download.txt", 
                 transfer_encoding="chunked", extras={}):
        super(CachedDownload, self).__init__(mimetype, content_disposition, 
                                             transfer_encoding, extras)
        self.cacheindex = cacheindex
        
    def get_content(self):
        return cache.get(self.cacheindex, None)

class FileDownload(DownloadBase):
    """
    Download that lives on the filesystem
    """
    
    def __init__(self, filename, mimetype="text/plain", 
                 content_disposition="attachment; filename=download.txt", 
                 transfer_encoding="chunked", extras={}):
        super(CachedDownload, self).__init__(mimetype, content_disposition, 
                                             transfer_encoding, extras)
        self.filename = filename
        
    def get_content(self):
        with open(self.filename, 'rb') as f:
            return f.read()
        <commit_msg>change default encodign across the board<commit_after>from django.core.cache import cache
from django.http import HttpResponse

class DownloadBase(object):
    """
    A basic download object.
    """
    
    def __init__(self, mimetype="text/plain", 
                 content_disposition="attachment; filename=download.txt", 
                 transfer_encoding=None, extras={}):
        self.mimetype = mimetype
        self.content_disposition = content_disposition
        self.transfer_encoding = transfer_encoding
        self.extras = {}
        
    def get_content(self):
        raise NotImplemented("Use CachedDownload or FileDownload!")
    
    def toHttpResponse(self):
        response = HttpResponse(self.get_content(), mimetype=self.mimetype)
        if self.transfer_encoding is not None:
            response['Transfer-Encoding'] = self.transfer_encoding
        response['Content-Disposition'] = self.content_disposition
        for k,v in self.extras.items():
            response[k] = v
        return response
    
    def __str__(self):
        return "content-type: %s, disposition: %s" % (self.mimetype, self.content_disposition)

class CachedDownload(DownloadBase):
    """
    Download that lives in the cache
    """
    
    def __init__(self, cacheindex, mimetype="text/plain", 
                 content_disposition="attachment; filename=download.txt", 
                 transfer_encoding=None, extras={}):
        super(CachedDownload, self).__init__(mimetype, content_disposition, 
                                             transfer_encoding, extras)
        self.cacheindex = cacheindex
        
    def get_content(self):
        return cache.get(self.cacheindex, None)

class FileDownload(DownloadBase):
    """
    Download that lives on the filesystem
    """
    
    def __init__(self, filename, mimetype="text/plain", 
                 content_disposition="attachment; filename=download.txt", 
                 transfer_encoding=None, extras={}):
        super(CachedDownload, self).__init__(mimetype, content_disposition, 
                                             transfer_encoding, extras)
        self.filename = filename
        
    def get_content(self):
        with open(self.filename, 'rb') as f:
            return f.read()
        
{'fraction_non_alphanumeric': 0.09467455621301775, 'fraction_numerical': 0.0, 'mean_word_length': 2.3362573099415203, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1407842', 'n_tokens_mistral': 1198, 'n_tokens_neox': 1157, 'n_words': 257}