<commit_before>__all__ = ('BlockStorageFile',)

import os
import struct
import logging

from pyoram.storage.block_storage import \
    (BlockStorageInterface,
     BlockStorageTypeFactory)

import six
from six.moves import xrange

log = logging.getLogger("pyoram")

class BlockStorageFile(BlockStorageInterface):

    _index_struct_string = "!LLL?"
    _index_offset = struct.calcsize(_index_struct_string)

    def __init__(self,
                 storage_name,
                 ignore_lock=False):
        self._ignore_lock = ignore_lock
        self._f = None
        if not os.path.exists(storage_name):
            raise IOError("Storage location does not exist: %s"
                          % (storage_name))
        self._storage_name = storage_name
        self._f = open(self.storage_name, "r+b")
        self._f.seek(0)
        self._block_size, self._block_count, user_header_size, locked = \
            struct.unpack(
                BlockStorageFile._index_struct_string,
                self._f.read(BlockStorageFile._index_offset))

        if locked and (not self._ignore_lock):
            self._f.close()
            self._f = None
            raise IOError(
                "Can not open block storage device because it is "
                "locked by another process. To ignore this check, "
                "initialize this class with the keyword 'ignore_lock' "
                "set to True.")
        self._user_header_data = bytes()
        if user_header_size > 0:
            self._user_header_data = \
                self._f.read(user_header_size)
        self._header_offset = BlockStorageFile._index_offset + \
                              len(self._user_header_data)
        if not self._ignore_lock:
            # turn on the locked flag
            self._f.seek(0)
            self._f.write(struct.pack(BlockStorageFile._index_struct_string,
                                      self.block_size,
                                      self.block_count,
                                      len(self._user_header_data),
                                      True))
            self._f.flush()

    #
    # Define BlockStorageInterface Methods
    #

    def clone_device(self):
        return BlockStorageFile(self.storage_name, ignore_lock=True)

    @classmethod
    def compute_storage_size(cls,
                             block_size,
                             block_count,
                             header_data=None,
                             ignore_header=False):
        assert (block_size > 0) and (block_size == int(block_size))
        assert (block_count > 0) and (block_count == int(block_count))<commit_msg>Adding workaround for strange Python3 behavior that occurs when using tree top caching with local file storage. I believe this is something broken with bufferedio in Python3.<commit_after>__all__ = ('BlockStorageFile',)

import os
import struct
import logging

from pyoram.storage.block_storage import \
    (BlockStorageInterface,
     BlockStorageTypeFactory)

import six
from six.moves import xrange

log = logging.getLogger("pyoram")

class BlockStorageFile(BlockStorageInterface):

    _index_struct_string = "!LLL?"
    _index_offset = struct.calcsize(_index_struct_string)

    def __init__(self,
                 storage_name,
                 ignore_lock=False):
        self._ignore_lock = ignore_lock
        self._f = None
        if not os.path.exists(storage_name):
            raise IOError("Storage location does not exist: %s"
                          % (storage_name))
        self._storage_name = storage_name
        self._f = open(self.storage_name, "r+b")
        self._f.seek(0)
        self._block_size, self._block_count, user_header_size, locked = \
            struct.unpack(
                BlockStorageFile._index_struct_string,
                self._f.read(BlockStorageFile._index_offset))

        if locked and (not self._ignore_lock):
            self._f.close()
            self._f = None
            raise IOError(
                "Can not open block storage device because it is "
                "locked by another process. To ignore this check, "
                "initialize this class with the keyword 'ignore_lock' "
                "set to True.")
        self._user_header_data = bytes()
        if user_header_size > 0:
            self._user_header_data = \
                self._f.read(user_header_size)
        self._header_offset = BlockStorageFile._index_offset + \
                              len(self._user_header_data)

        # TODO: Figure out why this is required for Python3
        #       in order to prevent issues with the
        #       TopCachedEncryptedHeapStorage class. The
        #       problem has something to do with bufferedio,
        #       but it makes no sense why this fixes it (all
        #       we've done is read above these lines). As
        #       part of this, investigate whethor or not we
        #       need the call to flush after write_block(s),
        #       or if its simply connected to some Python3
        #       bug in bufferedio.
        self._f.flush()

        if not self._ignore_lock:
            # turn on the locked flag
            self._f.seek(0)
            self._f.write(struct.pack(BlockStorageFile._index_struct_string,
                                      self.block_size,
                                      self.block_count,
                                      len(self._user_header_data),
                                      True))
            self._f.flush()

    #
    # Define BlockStorageInterface Methods
    #

    def clone_device(self):
        return BlockStorageFile(self.storage_name, ignore_lock=True)

    @classmethod
    def compute_storage_size(cls,
                             block_size,
                             block_count,
                             header_data=None,
                             ignore_header=False):
        assert (block_size > 0) and (block_size == int(block_size))
        assert (block_count > 0) and (block_count == int(block_count))
{'fraction_non_alphanumeric': 0.07136984048676205, 'fraction_numerical': 0.002302252918927808, 'mean_word_length': 1.7043130280124499, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6802714', 'n_tokens_mistral': 1605, 'n_tokens_neox': 1552, 'n_words': 392}