<commit_before>from django.conf import settings
from django.core.cache import cache
from django.core.files.storage import get_storage_class

from celery.registry import tasks
from celery.task import Task

MAX_RETRIES = getattr(settings, 'QUEUED_REMOTE_STORAGE_RETRIES', 5)
RETRY_DELAY = getattr(settings, 'QUEUED_REMOTE_STORAGE_RETRY_DELAY', 60)

class SaveToRemoteTask(Task):
    max_retries = MAX_RETRIES
    default_retry_delay = RETRY_DELAY

    def run(self, name, local, remote, cache_key, **kwargs):
        local_storage = get_storage_class(local)()
        remote_storage = get_storage_class(remote)()

        try:
            remote_storage.save(name, local_storage.open(name))
        except:
            # something went wrong while uploading the file, retry
            self.retry([name, local, remote, cache_key], **kwargs)
            return False

        cache.set(cache_key, True)
        return True

tasks.register(SaveToRemoteTask)<commit_msg>When a remote save attempt fails, log details of the problem.<commit_after>from django.conf import settings
from django.core.cache import cache
from django.core.files.storage import get_storage_class

from celery.registry import tasks
from celery.task import Task

MAX_RETRIES = getattr(settings, 'QUEUED_REMOTE_STORAGE_RETRIES', 5)
RETRY_DELAY = getattr(settings, 'QUEUED_REMOTE_STORAGE_RETRY_DELAY', 60)

class SaveToRemoteTask(Task):
    max_retries = MAX_RETRIES
    default_retry_delay = RETRY_DELAY

    def run(self, name, local, remote, cache_key, **kwargs):
        local_storage = get_storage_class(local)()
        remote_storage = get_storage_class(remote)()

        try:
            remote_storage.save(name, local_storage.open(name))
        except:
            # something went wrong while uploading the file, retry
            logger = self.get_logger(**kwargs)
            logger.exception("Unable to save '%s' to remote storage. About "
                    "to retry." % name)
            self.retry([name, local, remote, cache_key], **kwargs)
            return False

        cache.set(cache_key, True)
        return True

tasks.register(SaveToRemoteTask)
{'fraction_non_alphanumeric': 0.08356676003734827, 'fraction_numerical': 0.0028011204481792717, 'mean_word_length': 3.4460580912863072, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1515743', 'n_tokens_mistral': 657, 'n_tokens_neox': 631, 'n_words': 160}