<commit_before>from operator import itemgetter
from datetime import datetime, timedelta

from django.db import IntegrityError

from celery.decorators import periodic_task

from mygpo.data.podcast import calc_similar_podcasts
from mygpo.celery import celery
from mygpo.podcasts.models import Podcast

from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)


@celery.task
def update_podcasts(podcast_urls):
    """ Task to update a podcast """
    from mygpo.data.feeddownloader import update_podcasts as update
    podcasts = update(podcast_urls)
    return list(podcasts)


@celery.task
def update_related_podcasts(podcast, max_related=20):
    get_podcast = itemgetter(0)

    related = calc_similar_podcasts(podcast)[:max_related]
    related = map(get_podcast, related)

    for p in related:
        try:
            podcast.related_podcasts.add(p)
        except IntegrityError:
            logger.warn('Integrity error while adding related podcast',
                        exc_info=True)


# interval in which podcast updates are scheduled
UPDATE_INTERVAL = timedelta(hours=1)


@periodic_task(run_every=UPDATE_INTERVAL)
def schedule_updates(interval=UPDATE_INTERVAL):
    """ Schedules podcast updates that are due within ``interval`` """
    now = datetime.utcnow()

    # fetch podcasts for which an update is due within the next hour
    podcasts = Podcast.objects.all()\
                              .next_update_between(now, now+interval)\
                              .prefetch_related('urls')\
                              .only('pk')

    logger.error('Scheduling %d podcasts for update', podcasts.count())
    # queue all those podcast updates
    for podcast in podcasts:
        update_podcasts.delay([podcast.url])<commit_msg>Add upper bound for periodic scheduling of updates<commit_after>from operator import itemgetter
from datetime import datetime, timedelta

from django.db import IntegrityError

from celery.decorators import periodic_task

from mygpo.data.podcast import calc_similar_podcasts
from mygpo.celery import celery
from mygpo.podcasts.models import Podcast

from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)


@celery.task
def update_podcasts(podcast_urls):
    """ Task to update a podcast """
    from mygpo.data.feeddownloader import update_podcasts as update
    podcasts = update(podcast_urls)
    return list(podcasts)


@celery.task
def update_related_podcasts(podcast, max_related=20):
    get_podcast = itemgetter(0)

    related = calc_similar_podcasts(podcast)[:max_related]
    related = map(get_podcast, related)

    for p in related:
        try:
            podcast.related_podcasts.add(p)
        except IntegrityError:
            logger.warn('Integrity error while adding related podcast',
                        exc_info=True)


# interval in which podcast updates are scheduled
UPDATE_INTERVAL = timedelta(hours=1)


@periodic_task(run_every=UPDATE_INTERVAL)
def schedule_updates(interval=UPDATE_INTERVAL):
    """ Schedules podcast updates that are due within ``interval`` """
    now = datetime.utcnow()

    # max number of updates to schedule (one per minute)
    max_updates = UPDATE_INTERVAL.total_seconds() / 60

    # fetch podcasts for which an update is due within the next hour
    podcasts = Podcast.objects.all()\
                              .next_update_between(now, now+interval)\
                              .prefetch_related('urls')\
                              .only('pk')\
                              .limit(max_updates)

    logger.error('Scheduling %d podcasts for update', podcasts.count())
    # queue all those podcast updates
    for podcast in podcasts:
        update_podcasts.delay([podcast.url])
{'fraction_non_alphanumeric': 0.07628700986929848, 'fraction_numerical': 0.002667377967457989, 'mean_word_length': 3.171301446051168, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '467844', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1040, 'n_words': 289}