<commit_before>#
# Copyright (c) 2014 ThoughtWorks, Inc.
#
# Pixelated is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pixelated is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.

import io

import os
from whoosh.filedb.filestore import FileStorage
from whoosh.filedb.structfile import StructFile, BufferFile
from cryptography.fernet import Fernet
from whoosh.util import random_name


class EncryptedFileStorage(FileStorage):
    def __init__(self, path, masterkey=None):
        self.masterkey = masterkey
        self.f = Fernet(masterkey)
        self._tmp_storage = self.temp_storage
        self.length_cache = {}
        FileStorage.__init__(self, path, supports_mmap=False)

    def open_file(self, name, **kwargs):
        return self._open_encrypted_file(name, onclose=self._encrypt_index_on_close(name))

    def create_file(self, name, excl=False, mode="w+b", **kwargs):
        f = StructFile(io.BytesIO(), name=name, onclose=self._encrypt_index_on_close(name))
        f.is_real = False
        return f

    def temp_storage(self, name=None):
        name = name or "%s.tmp" % random_name()
        path = os.path.join(self.folder, name)
        return EncryptedFileStorage(path, self.masterkey).create()

    def file_length(self, name):
        return self.length_cache[name]

    def _encrypt_index_on_close(self, name):
        def wrapper(struct_file):
            struct_file.seek(0)
            content = struct_file.file.read()
            self.length_cache[name] = len(content)
            encrypted_content = self.f.encrypt(content)
            with open(self._fpath(name), 'w+b') as f:
                f.write(encrypted_content)
        return wrapper

    def _open_encrypted_file(self, name, onclose=lambda x: None):
        file_content = open(self._fpath(name), "rb").read()
        decrypted = self.f.decrypt(file_content)
        self.length_cache[name] = len(decrypted)
        return BufferFile(buffer(decrypted), name=name, onclose=onclose)
<commit_msg>related to #194 - solves a concurrency problem on encrypted whoosh caused by unnecessary writes on close callback.<commit_after>#
# Copyright (c) 2014 ThoughtWorks, Inc.
#
# Pixelated is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Pixelated is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Pixelated. If not, see <http://www.gnu.org/licenses/>.

import io
from hashlib import sha512

import os
from whoosh.filedb.filestore import FileStorage
from whoosh.filedb.structfile import StructFile, BufferFile
from cryptography.fernet import Fernet
from whoosh.util import random_name


class EncryptedFileStorage(FileStorage):
    def __init__(self, path, masterkey=None):
        self.masterkey = masterkey
        self.f = Fernet(masterkey)
        self._tmp_storage = self.temp_storage
        self.length_cache = {}
        FileStorage.__init__(self, path, supports_mmap=False)

    def open_file(self, name, **kwargs):
        return self._open_encrypted_file(name, onclose=self._encrypt_index_on_close(name))

    def create_file(self, name, excl=False, mode="w+b", **kwargs):
        f = StructFile(io.BytesIO(), name=name, onclose=self._encrypt_index_on_close(name))
        f.is_real = False
        return f

    def temp_storage(self, name=None):
        name = name or "%s.tmp" % random_name()
        path = os.path.join(self.folder, name)
        return EncryptedFileStorage(path, self.masterkey).create()

    def file_length(self, name):
        return self.length_cache[name][0]

    def _encrypt_index_on_close(self, name):
        def wrapper(struct_file):
            struct_file.seek(0)
            content = struct_file.file.read()
            file_hash = sha512(content).digest()
            if name in self.length_cache and file_hash == self.length_cache[name][1]:
                return
            self.length_cache[name] = (len(content), file_hash)
            encrypted_content = self.f.encrypt(content)
            with open(self._fpath(name), 'w+b') as f:
                f.write(encrypted_content)
        return wrapper

    def _open_encrypted_file(self, name, onclose=lambda x: None):
        file_content = open(self._fpath(name), "rb").read()
        decrypted = self.f.decrypt(file_content)
        self.length_cache[name] = (len(decrypted), sha512(decrypted).digest())
        return BufferFile(buffer(decrypted), name=name, onclose=onclose)

{'fraction_non_alphanumeric': 0.093505039193729, 'fraction_numerical': 0.0048525569242254575, 'mean_word_length': 3.414332784184514, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3743877', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1546, 'n_words': 470}