<commit_before>from StringIO import StringIO
import io
import sys
import errno
import os

from whoosh.filedb.filestore import FileStorage, ReadOnlyError

from whoosh.filedb.structfile import StructFile, BufferFile
from cryptography.fernet import Fernet
from whoosh.util import random_name


class EncryptedFileStorage(FileStorage):
    def __init__(self, path, masterkey=None):
        self.masterkey = masterkey
        self.f = Fernet(masterkey)
        self._tmp_storage = self.temp_storage
        FileStorage.__init__(self, path, supports_mmap=False)

    def open_file(self, name, **kwargs):
        def onclose(file_struct):
            file_struct.seek(0)
            content = file_struct.file.read()
            encrypted_content = self.f.encrypt(content)
            _file = open(self._fpath(name), 'w+b')
            _file.write(encrypted_content)

        return self._open_file(name, onclose=onclose)

    def create_file(self, name, excl=False, mode="w+b", **kwargs):
        def onclose(file_struct):
            file_struct.seek(0)
            content = file_struct.file.read()
            encrypted_content = self.f.encrypt(content)
            _file = open(self._fpath(name), 'w+b')
            _file.write(encrypted_content)

        f = StructFile(io.BytesIO(), name=name, onclose=onclose)
        f.is_real = False
        return f

    def temp_storage(self, name=None):
        name = name or "%s.tmp" % random_name()
        path = os.path.join(self.folder, name)<commit_msg>using with to write encrypted storage<commit_after>from StringIO import StringIO
import io
import sys
import errno
import os

from whoosh.filedb.filestore import FileStorage, ReadOnlyError

from whoosh.filedb.structfile import StructFile, BufferFile
from cryptography.fernet import Fernet
from whoosh.util import random_name


class EncryptedFileStorage(FileStorage):
    def __init__(self, path, masterkey=None):
        self.masterkey = masterkey
        self.f = Fernet(masterkey)
        self._tmp_storage = self.temp_storage
        FileStorage.__init__(self, path, supports_mmap=False)

    def open_file(self, name, **kwargs):
        def onclose(file_struct):
            file_struct.seek(0)
            content = file_struct.file.read()
            encrypted_content = self.f.encrypt(content)
            with open(self._fpath(name), 'w+b') as _f:
                _f.write(encrypted_content)

        return self._open_file(name, onclose=onclose)

    def create_file(self, name, excl=False, mode="w+b", **kwargs):
        def onclose(file_struct):
            file_struct.seek(0)
            content = file_struct.file.read()
            encrypted_content = self.f.encrypt(content)
            with open(self._fpath(name), 'w+b') as _f:
                _f.write(encrypted_content)

        f = StructFile(io.BytesIO(), name=name, onclose=onclose)
        f.is_real = False
        return f

    def temp_storage(self, name=None):
        name = name or "%s.tmp" % random_name()
        path = os.path.join(self.folder, name)
{'fraction_non_alphanumeric': 0.0995385629531971, 'fraction_numerical': 0.0013183915622940012, 'mean_word_length': 2.8613231552162848, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3013122', 'n_tokens_mistral': 937, 'n_tokens_neox': 916, 'n_words': 206}