<commit_before># -*- coding: utf8 -*-
import os

from datetime import datetime
from celery import task

from django.template.defaultfilters import slugify
from django.conf import settings

from video.models import Video
from video.utils import get_video_info, download_thumbnail, VideoDownloader, \
                        encode_videos


@task
def fetch_video(video_id):
    """Task to download the video.

    The video parameter is an object of type Video, as defined in
    megascops.models.Video """
    print "Starting download of video %s" % video_id
    video = Video.objects.get(pk=video_id)
    video.state = "FETCHING_INFO"
    video.save()
    vid_info = get_video_info(video.page_url)
    page_title = vid_info['pagetitle'] if vid_info['pagetitle']\
                    else "%s video %s" % (vid_info['hostid'],
                                        str(datetime.now()))
    video.page_title = page_title
    video.host = vid_info['hostid']

    flv_url = vid_info['mediaurl']
    filename = slugify(page_title)
    video.filename = filename
    video.duration = vid_info['mediaduration']
    video.file_suffix = vid_info['filesuffix']
    dest_file = "%s.%s" % (filename,
                            vid_info['filesuffix'])
    dest_path = os.path.join(settings.MEDIA_ROOT, 'video/', dest_file)
    if vid_info['mediathumbnail']:
        thumbnail = download_thumbnail(vid_info)
        video.thumbnail = thumbnail
    video.state = "DOWNLOAD_STARTED"
    video.save()
    downloader = VideoDownloader(video_id)
    downloader.download(flv_url, dest_path)<commit_msg>Cleanup video.tasks<commit_after># -*- coding: utf8 -*-
import os

from datetime import datetime
from celery import task

from django.template.defaultfilters import slugify
from django.conf import settings

from video.models import Video
from video.utils import (get_video_info, download_thumbnail, VideoDownloader,
                         encode_videos)


@task
def fetch_video(video_id):
    """Task to download the video.

    The video parameter is an object of type Video, as defined in
    megascops.models.Video """
    video = Video.objects.get(pk=video_id)
    video.state = "FETCHING_INFO"
    video.save()
    vid_info = get_video_info(video.page_url)
    page_title = (
        vid_info['pagetitle']
        if vid_info['pagetitle']
        else "%s video %s" % (vid_info['hostid'], str(datetime.now()))
    )
    video.page_title = page_title
    video.host = vid_info['hostid']

    flv_url = vid_info['mediaurl']
    filename = slugify(page_title)
    video.filename = filename
    video.duration = vid_info['mediaduration']
    video.file_suffix = vid_info['filesuffix']
    dest_file = "%s.%s" % (filename, vid_info['filesuffix'])
    dest_path = os.path.join(settings.MEDIA_ROOT, 'video/', dest_file)
    if vid_info['mediathumbnail']:
        thumbnail = download_thumbnail(vid_info)
        video.thumbnail = thumbnail
    video.state = "DOWNLOAD_STARTED"
    video.save()
    downloader = VideoDownloader(video_id)
    downloader.download(flv_url, dest_path)
{'fraction_non_alphanumeric': 0.10479921645445642, 'fraction_numerical': 0.000652954619653934, 'mean_word_length': 3.2378976486860305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5581593', 'n_tokens_mistral': 958, 'n_tokens_neox': 932, 'n_words': 217}