<commit_before># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.
import logging
import urllib
from urllib import urlopen

from celery.decorators import periodic_task
from celery.schedules import crontab, timedelta
from celery.signals import task_failure
from celery.task import task
from django.conf import settings
from django.contrib.sites.models import Site
from django.core.files.base import ContentFile
from django.db.models import ObjectDoesNotExist
from django.utils import simplejson as json
from django.utils.http import urlquote_plus
from haystack import site
from raven.contrib.django.models import client

from messages.models import Message
from utils import send_templated_email, DEFAULT_PROTOCOL
from utils.metrics import Gauge, Meter
from videos.models import VideoFeed, SubtitleLanguage, Video, Subtitle, SubtitleVersion
from videos.feed_parser import FeedParser

celery_logger = logging.getLogger('celery.task')


def process_failure_signal(exception, traceback, sender, task_id,
                           signal, args, kwargs, einfo, **kw):
    exc_info = (type(exception), exception, traceback)
    try:
        celery_logger.error(
            'Celery job exception: %s(%s)' % (exception.__class__.__name__, exception),
            exc_info=exc_info,
            extra={
                'data': {
                    'task_id': task_id,
                    'sender': sender,
                    'args': args,
                    'kwargs': kwargs,
                }
            }
        )
    except:
        pass
task_failure.connect(process_failure_signal)

@periodic_task(run_every=crontab(hour=3, day_of_week=1))
def cleanup():
    import datetime
    from django.db import transaction
    from django.contrib.sessions.models import Session
    from djcelery.models import TaskState
    from auth.models import EmailConfirmation

    EmailConfirmation.objects.delete_expired_confirmations()

    now = datetime.datetime.now()
    Session.objects.filter(expire_date__lt=now).delete()

    d = now - datetime.timedelta(days=31)
    TaskState.objects.filter(tstamp__lt=d).delete()
    transaction.commit_unless_managed()

@task
def save_thumbnail_in_s3(video_id):
    try:
        video = Video.objects.get(pk=video_id)
    except Video.DoesNotExist:
        return

    if video.thumbnail and not video.s3_thumbnail:
        content = ContentFile(urlopen(video.thumbnail).read())
        video.s3_thumbnail.save(video.thumbnail.split('/')[-1], content)

@periodic_task(run_every=crontab(minute=0, hour=1))
def update_from_feed(*args, **kwargs):
    for feed in VideoFeed.objects.all():
        update_video_feed.delay(feed.pk)

@task
def update_subtitles_fetched_counter_for_sl(sl_pk):
    try:
        sl = SubtitleLanguage.objects.get(pk=sl_pk)
        sl.subtitles_fetched_counter.incr()
    except (SubtitleLanguage.DoesNotExist, ValueError):
        return

@task
def update_video_feed(video_feed_id):
    try:
        video_feed = VideoFeed.objects.get(pk=video_feed_id)
        video_feed.update()
    except VideoFeed:
        msg = '**update_video_feed**. VideoFeed does not exist. ID: %s' % video_feed_id
        client.captureMessage(msg)

@task()
def video_changed_tasks(video_pk, new_version_id=None):
    from videos import metadata_manager
    from videos.models import Video
    from teams.models import TeamVideo
    metadata_manager.update_metadata(video_pk)
    if new_version_id is not None:
        _send_notification(new_version_id)
        _check_alarm(new_version_id)
        _detect_language(new_version_id)
        _update_captions_in_original_service(new_version_id)

    video = Video.objects.get(pk=video_pk)

    tv = video.get_team_video()
    if tv:
        tv_search_index = site.get_index(TeamVideo)
        tv_search_index.backend.update(tv_search_index, [tv])

    video.update_search_index()

@task()
def send_change_title_email(video_id, user_id, old_title, new_title):
    from videos.models import Video
    from auth.models import CustomUser as User

    domain = Site.objects.get_current().domain

    try:
        video = Video.objects.get(id=video_id)
        user = user_id and User.objects.get(id=user_id)
    except ObjectDoesNotExist:
        return
<commit_msg>Changed imports to match the new datamodel<commit_after># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.
import logging
import urllib
from urllib import urlopen

from celery.decorators import periodic_task
from celery.schedules import crontab, timedelta
from celery.signals import task_failure
from celery.task import task
from django.conf import settings
from django.contrib.sites.models import Site
from django.core.files.base import ContentFile
from django.db.models import ObjectDoesNotExist
from django.utils import simplejson as json
from django.utils.http import urlquote_plus
from haystack import site
from raven.contrib.django.models import client

from messages.models import Message
from utils import send_templated_email, DEFAULT_PROTOCOL
from utils.metrics import Gauge, Meter
from videos.models import VideoFeed, Video
from subtitles.models import SubtitleLanguage, SubtitleVersion
from videos.feed_parser import FeedParser

celery_logger = logging.getLogger('celery.task')

def process_failure_signal(exception, traceback, sender, task_id,
                           signal, args, kwargs, einfo, **kw):
    exc_info = (type(exception), exception, traceback)
    try:
        celery_logger.error(
            'Celery job exception: %s(%s)' % (exception.__class__.__name__, exception),
            exc_info=exc_info,
            extra={
                'data': {
                    'task_id': task_id,
                    'sender': sender,
                    'args': args,
                    'kwargs': kwargs,
                }
            }
        )
    except:
        pass
task_failure.connect(process_failure_signal)

@periodic_task(run_every=crontab(hour=3, day_of_week=1))
def cleanup():
    import datetime
    from django.db import transaction
    from django.contrib.sessions.models import Session
    from djcelery.models import TaskState
    from auth.models import EmailConfirmation

    EmailConfirmation.objects.delete_expired_confirmations()

    now = datetime.datetime.now()
    Session.objects.filter(expire_date__lt=now).delete()

    d = now - datetime.timedelta(days=31)
    TaskState.objects.filter(tstamp__lt=d).delete()
    transaction.commit_unless_managed()

@task
def save_thumbnail_in_s3(video_id):
    try:
        video = Video.objects.get(pk=video_id)
    except Video.DoesNotExist:
        return

    if video.thumbnail and not video.s3_thumbnail:
        content = ContentFile(urlopen(video.thumbnail).read())
        video.s3_thumbnail.save(video.thumbnail.split('/')[-1], content)

@periodic_task(run_every=crontab(minute=0, hour=1))
def update_from_feed(*args, **kwargs):
    for feed in VideoFeed.objects.all():
        update_video_feed.delay(feed.pk)

@task
def update_subtitles_fetched_counter_for_sl(sl_pk):
    try:
        sl = SubtitleLanguage.objects.get(pk=sl_pk)
        sl.subtitles_fetched_counter.incr()
    except (SubtitleLanguage.DoesNotExist, ValueError):
        return

@task
def update_video_feed(video_feed_id):
    try:
        video_feed = VideoFeed.objects.get(pk=video_feed_id)
        video_feed.update()
    except VideoFeed:
        msg = '**update_video_feed**. VideoFeed does not exist. ID: %s' % video_feed_id
        client.captureMessage(msg)

@task()
def video_changed_tasks(video_pk, new_version_id=None):
    from videos import metadata_manager
    from videos.models import Video
    from teams.models import TeamVideo
    metadata_manager.update_metadata(video_pk)
    if new_version_id is not None:
        _send_notification(new_version_id)
        _check_alarm(new_version_id)
        _detect_language(new_version_id)
        _update_captions_in_original_service(new_version_id)

    video = Video.objects.get(pk=video_pk)

    tv = video.get_team_video()

    if tv:
        tv_search_index = site.get_index(TeamVideo)
        tv_search_index.backend.update(tv_search_index, [tv])

    video.update_search_index()

@task()
def send_change_title_email(video_id, user_id, old_title, new_title):
    from videos.models import Video
    from auth.models import CustomUser as User

    domain = Site.objects.get_current().domain

    try:
        video = Video.objects.get(id=video_id)
        user = user_id and User.objects.get(id=user_id)
    except ObjectDoesNotExist:
        return

{'fraction_non_alphanumeric': 0.08005236129292116, 'fraction_numerical': 0.0034236229986909675, 'mean_word_length': 3.5643382352941178, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3907160', 'n_tokens_mistral': 2968, 'n_tokens_neox': 2910, 'n_words': 784}