<commit_before>from collections import namedtuple
import os
from tempfile import mkstemp
import uuid
from django.core.cache import caches
from corehq.apps.case_importer.tracking.models import CaseUploadFileMeta
from corehq.blobs import get_blob_db
from corehq.blobs.util import random_url_id
from corehq.util.files import file_extention_from_filename
from dimagi.utils.decorators.memoized import memoized

BUCKET_PREFIX = 'case_importer'


class PersistentFileStore(object):
    def __init__(self, bucket, meta_model):
        """
        :meta_model is a django model used to store meta info
            must contain columns identifier, filename, length
        """

        self._bucket_prefix = bucket
        self._db = get_blob_db()
        self._meta_model = meta_model

    def _get_bucket(self, padding):
        return '{}/{}'.format(self._bucket_prefix, padding)

    def write_file(self, f, filename):
        padding = random_url_id(16)
        blob_info = self._db.put(f, bucket=self._get_bucket(padding))
        identifier = '{}/{}'.format(padding, blob_info.identifier)
        file_meta = self._meta_model(identifier=identifier, filename=filename,
                                     length=blob_info.length)
        file_meta.save()
        return file_meta

    @memoized
    def get_tempfile_ref_for_contents(self, identifier):
        filename = self.get_filename(identifier)
        suffix = file_extention_from_filename(filename)
        padding, blob_identifier = identifier.split('/')
        content = self._db.get(blob_identifier, bucket=self._get_bucket(padding)).read()
        return make_temp_file(content, suffix)

    @memoized
    def get_filename(self, identifier):
        return self._meta_model.objects.values_list('filename', flat=True).get(identifier=identifier)


FileMeta = namedtuple('FileMeta', ['identifier', 'filename', 'length'])


class TransientFileStore(object):
    def __init__(self, bucket, timeout):
        self._bucket = bucket
        self._cache = caches['default']
        self._timeout = timeout

    def _get_key(self, identifier):
        return '{}/{}'.format(self._bucket, identifier)

    def _get_filename_key(self, identifier):
        return '{}/{}/filename'.format(self._bucket, identifier)

    def write_file(self, f, filename):
        identifier = str(uuid.uuid4())
        contents = f.read()
        content_length = len(contents)
        self._cache.set(self._get_key(identifier), (filename, contents), timeout=self._timeout)
        return FileMeta(identifier=identifier, filename=filename, length=content_length)

    def get_tempfile_ref_for_contents(self, identifier):
        try:
            filename, content = self._get_filename_content(identifier)
        except (TypeError, ValueError):
            return None
        suffix = file_extention_from_filename(filename)
        return make_temp_file(content, suffix)

    def get_filename(self, identifier):
        filename, _ = self._get_filename_content(identifier)
        return filename<commit_msg>add docstrings to *FileStore<commit_after>from collections import namedtuple
import os
from tempfile import mkstemp
import uuid
from django.core.cache import caches
from corehq.apps.case_importer.tracking.models import CaseUploadFileMeta
from corehq.blobs import get_blob_db
from corehq.blobs.util import random_url_id
from corehq.util.files import file_extention_from_filename
from dimagi.utils.decorators.memoized import memoized

BUCKET_PREFIX = 'case_importer'


class PersistentFileStore(object):
    """
    helper for writing and reading back arbitrary files to HIPAA-compliant storage
    (Riak or filesystem, via BlobDB)

    metadata is also stored in a django model given by meta_model,
    which is returned by write_file

    """
    def __init__(self, bucket, meta_model):
        """
        :meta_model is a django model used to store meta info
            must contain columns identifier, filename, length
        """

        self._bucket_prefix = bucket
        self._db = get_blob_db()
        self._meta_model = meta_model

    def _get_bucket(self, padding):
        return '{}/{}'.format(self._bucket_prefix, padding)

    def write_file(self, f, filename):
        padding = random_url_id(16)
        blob_info = self._db.put(f, bucket=self._get_bucket(padding))
        identifier = '{}/{}'.format(padding, blob_info.identifier)
        file_meta = self._meta_model(identifier=identifier, filename=filename,
                                     length=blob_info.length)
        file_meta.save()
        return file_meta

    @memoized
    def get_tempfile_ref_for_contents(self, identifier):
        filename = self.get_filename(identifier)
        suffix = file_extention_from_filename(filename)
        padding, blob_identifier = identifier.split('/')
        content = self._db.get(blob_identifier, bucket=self._get_bucket(padding)).read()
        return make_temp_file(content, suffix)

    @memoized
    def get_filename(self, identifier):
        return self._meta_model.objects.values_list('filename', flat=True).get(identifier=identifier)


FileMeta = namedtuple('FileMeta', ['identifier', 'filename', 'length'])


class TransientFileStore(object):
    """
    helper for writing and reading back arbitrary files to the cache

    """
    def __init__(self, bucket, timeout):
        self._bucket = bucket
        self._cache = caches['default']
        self._timeout = timeout

    def _get_key(self, identifier):
        return '{}/{}'.format(self._bucket, identifier)

    def _get_filename_key(self, identifier):
        return '{}/{}/filename'.format(self._bucket, identifier)

    def write_file(self, f, filename):
        identifier = str(uuid.uuid4())
        contents = f.read()
        content_length = len(contents)
        self._cache.set(self._get_key(identifier), (filename, contents), timeout=self._timeout)
        return FileMeta(identifier=identifier, filename=filename, length=content_length)

    def get_tempfile_ref_for_contents(self, identifier):
        try:
            filename, content = self._get_filename_content(identifier)
        except (TypeError, ValueError):
            return None
        suffix = file_extention_from_filename(filename)
        return make_temp_file(content, suffix)

    def get_filename(self, identifier):
        filename, _ = self._get_filename_content(identifier)
        return filename
{'fraction_non_alphanumeric': 0.09442193829853537, 'fraction_numerical': 0.0009348706762231225, 'mean_word_length': 3.363698164513936, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1454232', 'n_tokens_mistral': 1794, 'n_tokens_neox': 1762, 'n_words': 417}