<commit_before>import os
import plowshare
import json

import file_database
import storage
import transfer_meter
import payload
import helpers

class CloudManager(object):
    """Manages files uploaded to cloud services.

    This cloud manager uses plowshare to upload files
    to file locker providers, and keeps track of them
    in a local database. It also keeps a local cache.

    """
    RedundancyLevel = 3

    Plowshare = plowshare.Plowshare
    Database  = file_database.FileDatabase
    Storage   = storage.Storage
    Meter     = transfer_meter.TransferMeter

    def __init__(self, database_path, storage_path, storage_size):
        """Initialize a CloudManager instance.

        Arguments:
        database_path -- Path to a sqlite3 database
        storage_path  -- Path to the local cache directory
        storage_size  -- Maximum number of bytes to keep in local cache

        """
        self.file_database = self.Database(database_path)
        self.plowshare     = self.Plowshare()
        self.storage       = self.Storage(storage_path, storage_size)
        self.meter         = self.Meter(database_path)

    def close(self):
        """Clean up all resources."""
        self.file_database.close()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def upload(self, file_path, token):
        """Add a file to the cloud.

        If the given file didn't exist yet, this method
        makes room for it (by deleting older cached files)
        and then uploads it to three different cloud hosts.

        """
        key     = self.add_to_storage(file_path, token)
        uploads = self.plowshare.upload(file_path, self.RedundancyLevel)

        self.on_upload_finished(key, uploads)
        return key

    def add_to_storage(self, file_path, token):
        """Add a file to local storage.

        This method also registers the file in the database,
        without any payload information.

        """
        key = helpers.sha256(file_path)
        if self.exists(key):
            return key

        needed = os.path.getsize(file_path)

        if not self.make_room_for(needed):
            return False

        saved_path = self.storage.add(file_path, key)
        self.file_database.store(key, needed, saved_path, None, token)
        self.meter.measure_upload(needed)
        return key

    def on_upload_finished(self, key, uploads):
        record = self.file_database.fetch(key)

        info = json.dumps(payload.to_dict(payload.build(
            record.name,
            record.hash,
            record.size,
            uploads)))

        self.file_database.set_payload(key, info)

    def warm_up(self, file_hash):
        """Warm up the cache for the given hash

        This method makes room for the given file,
        and then fetches it from one of the cloud hosts.
        It returns the full path of the file.

        """
        record = self.file_database.fetch(file_hash)
        if record is None:
            return None

        if self.storage.is_cached(record.name):
            return self.storage.path(record.name)
<commit_msg>Fixes a flow bug in upload<commit_after>import os
import plowshare
import json

import file_database
import storage
import transfer_meter
import payload
import helpers

class CloudManager(object):
    """Manages files uploaded to cloud services.

    This cloud manager uses plowshare to upload files
    to file locker providers, and keeps track of them
    in a local database. It also keeps a local cache.

    """
    RedundancyLevel = 3

    Plowshare = plowshare.Plowshare
    Database  = file_database.FileDatabase
    Storage   = storage.Storage
    Meter     = transfer_meter.TransferMeter

    def __init__(self, database_path, storage_path, storage_size):
        """Initialize a CloudManager instance.

        Arguments:
        database_path -- Path to a sqlite3 database
        storage_path  -- Path to the local cache directory
        storage_size  -- Maximum number of bytes to keep in local cache

        """
        self.file_database = self.Database(database_path)
        self.plowshare     = self.Plowshare()
        self.storage       = self.Storage(storage_path, storage_size)
        self.meter         = self.Meter(database_path)

    def close(self):
        """Clean up all resources."""
        self.file_database.close()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def upload(self, file_path, token):
        """Add a file to the cloud.

        If the given file didn't exist yet, this method
        makes room for it (by deleting older cached files)
        and then uploads it to three different cloud hosts.

        """
        key = helpers.sha256(file_path)
        if self.exists(key):
            return key

        self.add_to_storage(key, file_path, token)
        uploads = self.plowshare.upload(file_path, self.RedundancyLevel)

        self.on_upload_finished(key, uploads)
        return key

    def add_to_storage(self, key, file_path, token):
        """Add a file to local storage.

        This method also registers the file in the database,
        without any payload information.

        """
        needed = os.path.getsize(file_path)

        if not self.make_room_for(needed):
            return False

        saved_path = self.storage.add(file_path, key)
        self.file_database.store(key, needed, saved_path, None, token)
        self.meter.measure_upload(needed)
        return saved_path

    def on_upload_finished(self, key, uploads):
        record = self.file_database.fetch(key)

        info = json.dumps(payload.to_dict(payload.build(
            record.name,
            record.hash,
            record.size,
            uploads)))

        self.file_database.set_payload(key, info)

    def warm_up(self, file_hash):
        """Warm up the cache for the given hash

        This method makes room for the given file,
        and then fetches it from one of the cloud hosts.
        It returns the full path of the file.

        """
        record = self.file_database.fetch(file_hash)
        if record is None:
            return None

        if self.storage.is_cached(record.name):
            return self.storage.path(record.name)

{'fraction_non_alphanumeric': 0.0740682248894504, 'fraction_numerical': 0.0015792798483891346, 'mean_word_length': 2.2915800415800414, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2824004', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1705, 'n_words': 556}