<commit_before>from celery.decorators import task
from celery.task.sets import subtask
from celery.utils.log import get_task_logger

from models import MediaBase, Video


logger = get_task_logger(__name__)


@task(max_retries=3)
def generate_thumbnail(video_id, callback=None):
    media = Video.objects.get(pk=video_id)
    logger.info("Generating thumbnail for %s" % media)
    try:
        media.make_thumbnail()
    except Exception, exc:
        logger.info("Generate thumbnail failed for %s - retrying " % media)
        raise generate_thumbnail.retry(exc=exc, countdown=60)
    logger.info("Done generating thumbnail for %s" % media)
    if callback:
        subtask(callback).delay(media.id)


@task(max_retries=3)
def encode_media(media_id, callback=None):
    base = MediaBase.objects.get(pk=media_id)
    media = base.get_media()
    logger.info("Encoding %s" % media)
    try:
        media.encode_file()
    except Exception as exc:
        logger.info("Encode Media failed for %s - retrying " % media)
        raise encode_media.retry(exc=exc, countdown=60)
    logger.info("Done encoding for %s" % media)
    logger.info("Saving model and starting upload for %s" % media)
    if callback:
        subtask(callback).delay(media.id)


@task(max_retries=3)
def upload_media(media_id):
    base = MediaBase.objects.get(pk=media_id)
    media = base.get_media()
    logger.info("Uploading encoded file for %s" % media)
    try:
        media.upload_file()
    except Exception as exc:
        logger.info("Upload media failed for %s - retrying " % media)
        raise upload_media.retry(exc=exc, countdown=60)
    logger.info("Upload complete for %s" % media)
<commit_msg>Remove unused callback from thumbnail task<commit_after>from celery.decorators import task
from celery.task.sets import subtask
from celery.utils.log import get_task_logger

from models import MediaBase, Video


logger = get_task_logger(__name__)


@task(max_retries=3)
def generate_thumbnail(video_id):
    media = Video.objects.get(pk=video_id)
    logger.info("Generating thumbnail for %s" % media)
    try:
        media.make_thumbnail()
    except Exception as exc:
        logger.info("Generate thumbnail failed for %s - retrying " % media)
        raise generate_thumbnail.retry(exc=exc, countdown=60)
    logger.info("Done generating thumbnail for %s" % media)


@task(max_retries=3)
def encode_media(media_id, callback=None):
    base = MediaBase.objects.get(pk=media_id)
    media = base.get_media()
    logger.info("Encoding %s" % media)
    try:
        media.encode_file()
    except Exception as exc:
        logger.info("Encode Media failed for %s - retrying " % media)
        raise encode_media.retry(exc=exc, countdown=60)
    logger.info("Done encoding for %s" % media)
    logger.info("Saving model and starting upload for %s" % media)
    if callback:
        subtask(callback).delay(media.id)


@task(max_retries=3)
def upload_media(media_id):
    base = MediaBase.objects.get(pk=media_id)
    media = base.get_media()
    logger.info("Uploading encoded file for %s" % media)
    try:
        media.upload_file()
    except Exception as exc:
        logger.info("Upload media failed for %s - retrying " % media)
        raise upload_media.retry(exc=exc, countdown=60)
    logger.info("Upload complete for %s" % media)

{'fraction_non_alphanumeric': 0.10773397532350286, 'fraction_numerical': 0.005416792055371652, 'mean_word_length': 3.6102635228848823, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '875699', 'n_tokens_mistral': 1055, 'n_tokens_neox': 1040, 'n_words': 264}