<commit_before>import os
import plowshare

import file_database
import storage
import transfer_meter
import helpers

class CloudManager(object):
    """Manages files uploaded to cloud services.

    This cloud manager uses plowshare to upload files
    to file locker providers, and keeps track of them
    in a local database. It also keeps a local cache.

    """
    Plowshare = plowshare.Plowshare
    Database  = file_database.FileDatabase
    Storage   = storage.Storage
    Meter     = transfer_meter.TransferMeter

    def __init__(self, database_path, storage_path, storage_size):
        """Initialize a CloudManager instance.

        Arguments:
        database_path -- Path to a sqlite3 database
        storage_path  -- Path to the local cache directory
        storage_size  -- Maximum number of bytes to keep in local cache

        """
        self.file_database = self.Database(database_path)
        self.plowshare     = self.Plowshare()
        self.storage       = self.Storage(storage_path, storage_size)
        self.meter         = self.Meter(database_path)

    def close(self):
        """Clean up all resources."""
        self.file_database.close()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def upload(self, file_path):
        """Add a file to the cloud.

        If the given file didn't exist yet, this method
        makes room for it (by deleting older cached files)
        and then uploads it to three different cloud hosts.
        """
        # Check if file exists
        key = helpers.sha256(file_path)
        if self.exists(key):
            return False

        needed = os.path.getsize(file_path)

        if self.make_room_for(needed) == False:
            return False

        info = self.plowshare.upload(file_path, 3)
        saved_path = self.storage.add(file_path, key)
        self.file_database.store(saved_path, info, True)
        self.meter.measure_upload(needed)
        return key

    def download(self, file_hash):
        """Warm up the cache for the given hash.

        Same as warm_up.

        """
        return self.warm_up(file_hash)

    def warm_up(self, file_hash):
        """Warm up the cache for the given hash

        This method makes room for the given file,
        and then fetches it from one of the cloud hosts.
        It returns the full path of the file.

        """
        record = self.file_database.fetch(file_hash)
        if record is None:
            return None

        if record.is_cached:
            return self.storage.path(record.name)

        self.make_room_for(record.size)

        # Plowshare shouldn't determine the filename.
        ret = self.plowshare.download(record.payload, self.storage.storage_path)

        self.file_database.restored_to_cache(record.hash)
        self.meter.measure_upload(record.size)

        return self.storage.path(record.name)


    def used_space(self):
        return self.storage.used()

    def usage_ratio(self):
        """Return the percentage of used space."""
        return 1.0 * self.storage.used() / self.storage.size()
<commit_msg>Style change<commit_after>import os
import plowshare

import file_database
import storage
import transfer_meter
import helpers

class CloudManager(object):
    """Manages files uploaded to cloud services.

    This cloud manager uses plowshare to upload files
    to file locker providers, and keeps track of them
    in a local database. It also keeps a local cache.

    """
    Plowshare = plowshare.Plowshare
    Database  = file_database.FileDatabase
    Storage   = storage.Storage
    Meter     = transfer_meter.TransferMeter

    def __init__(self, database_path, storage_path, storage_size):
        """Initialize a CloudManager instance.

        Arguments:
        database_path -- Path to a sqlite3 database
        storage_path  -- Path to the local cache directory
        storage_size  -- Maximum number of bytes to keep in local cache

        """
        self.file_database = self.Database(database_path)
        self.plowshare     = self.Plowshare()
        self.storage       = self.Storage(storage_path, storage_size)
        self.meter         = self.Meter(database_path)

    def close(self):
        """Clean up all resources."""
        self.file_database.close()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def upload(self, file_path):
        """Add a file to the cloud.

        If the given file didn't exist yet, this method
        makes room for it (by deleting older cached files)
        and then uploads it to three different cloud hosts.
        """
        # Check if file exists
        key = helpers.sha256(file_path)
        if self.exists(key):
            return False

        needed = os.path.getsize(file_path)

        if not self.make_room_for(needed):
            return False

        info = self.plowshare.upload(file_path, 3)
        saved_path = self.storage.add(file_path, key)
        self.file_database.store(saved_path, info, True)
        self.meter.measure_upload(needed)
        return key

    def warm_up(self, file_hash):
        """Warm up the cache for the given hash

        This method makes room for the given file,
        and then fetches it from one of the cloud hosts.
        It returns the full path of the file.

        """
        record = self.file_database.fetch(file_hash)
        if record is None:
            return None

        if record.is_cached:
            return self.storage.path(record.name)

        self.make_room_for(record.size)

        # Plowshare shouldn't determine the filename.
        ret = self.plowshare.download(record.payload, self.storage.storage_path)

        self.file_database.restored_to_cache(record.hash)
        self.meter.measure_upload(record.size)

        return self.storage.path(record.name)

    def download(self, file_hash):
        """Same as warm_up."""
        return self.warm_up(file_hash)


    def used_space(self):
        return self.storage.used()

    def usage_ratio(self):
        """Return the percentage of used space."""
        return 1.0 * self.storage.used() / self.storage.size()

{'fraction_non_alphanumeric': 0.07912052640025678, 'fraction_numerical': 0.0022468303643074948, 'mean_word_length': 2.3201917954182205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5497224', 'n_tokens_mistral': 1779, 'n_tokens_neox': 1690, 'n_words': 535}