<commit_before># -*- coding: utf-8 -*-
"""
Copyright 2013-2014 <NAME> <<EMAIL>>.

This file is part of the 1flow project.

1flow is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of
the License, or (at your option) any later version.

1flow is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public
License along with 1flow.  If not, see http://www.gnu.org/licenses/
"""

import logging
import time as pytime

from random import randrange
from constance import config

from celery import task

from django.utils.translation import ugettext_lazy as _

from ..models.reldb import (
    MailAccount,
    BaseFeed, basefeed_refresh_task
)

from oneflow.base.utils import RedisExpiringLock
from oneflow.base.utils.dateutils import (now, today, timedelta,
                                          naturaldelta, benchmark)

from gr_import import clean_gri_keys

from mongo import refresh_all_mongo_feeds

# Import this one, so that celery can find it,
# Else it complains about a missing import.
from checks import global_checker_task  # NOQA

LOGGER = logging.getLogger(__name__)


@task(queue='clean')
def clean_obsolete_redis_keys():
    """ Call in turn all redis-related cleaners. """

    start_time = pytime.time()

    if today() <= (config.GR_END_DATE + timedelta(days=1)):
        clean_gri_keys()

    LOGGER.info(u'clean_obsolete_redis_keys(): finished in %s.',
                naturaldelta(pytime.time() - start_time))


@task(queue='high')
def refresh_all_feeds(limit=None, force=False):
    """ Refresh all feeds (RSS/Mail/Twitter…). """

    if config.FEED_FETCH_DISABLED:
        # Do not raise any .retry(), this is a scheduled task.
        LOGGER.warning(u'Feed refresh disabled in configuration.')
        return

    # TODO: WIPE THIS when Mongo → PG migration is complete!
    refresh_all_mongo_feeds.delay()

    # Be sure two refresh operations don't overlap, but don't hold the
    # lock too long if something goes wrong. In production conditions
    # as of 20130812, refreshing all feeds takes only a moment:
    # [2013-08-12 09:07:02,028: INFO/MainProcess] Task
    #       oneflow.core.tasks.refresh_all_feeds succeeded in 1.99886608124s.
    my_lock = RedisExpiringLock('refresh_all_feeds', expire_time=120)

    if not my_lock.acquire():
        if force:
            my_lock.release()
            my_lock.acquire()
            LOGGER.warning(_(u'Forcing all feed refresh…'))

        else:
            # Avoid running this task over and over again in the queue
            # if the previous instance did not yet terminate. Happens
            # when scheduled task runs too quickly.
            LOGGER.warning(u'refresh_all_feeds() is already locked, aborting.')
            return

    # This should bring us a Polymorphic Query to refresh all feeds types.
    feeds = BaseFeed.objects.filter(is_active=True, is_internal=False)

    if limit:
        feeds = feeds.limit(limit)

    with benchmark('refresh_all_feeds()'):

        try:
            count = 0
            mynow = now()

            for feed in feeds:

                if feed.refresh_lock.is_locked():
                    LOGGER.info(u'Feed %s already locked, skipped.', feed)
                    continue

                interval = timedelta(seconds=feed.fetch_interval)

                if feed.last_fetch is None:

                    basefeed_refresh_task.delay(feed.id)

                    LOGGER.info(u'Launched immediate refresh of feed %s which '
                                u'has never been refreshed.', feed)

                elif force or feed.last_fetch + interval < mynow:

                    how_late = feed.last_fetch + interval - mynow
                    how_late = how_late.days * 86400 + how_late.seconds

                    if config.FEED_REFRESH_RANDOMIZE:
                        countdown = randrange(
                            config.FEED_REFRESH_RANDOMIZE_DELAY)

                        basefeed_refresh_task.apply_async((feed.id, force),
                                                          countdown=countdown)

                    else:
                        countdown = 0
                        basefeed_refresh_task.delay(feed.id, force)

                    LOGGER.info(u'%s refresh of feed %s %s (%s late).',
                                u'Scheduled randomized'
                                if countdown else u'Launched',
                                feed,
                                u' in {0}'.format(naturaldelta(countdown))<commit_msg>fix the relational fetcher task.<commit_after># -*- coding: utf-8 -*-
"""
Copyright 2013-2014 <NAME> <<EMAIL>>.

This file is part of the 1flow project.

1flow is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of
the License, or (at your option) any later version.

1flow is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public
License along with 1flow.  If not, see http://www.gnu.org/licenses/
"""

import logging
import time as pytime

from random import randrange
from constance import config

from celery import task

from django.utils.translation import ugettext_lazy as _

from ..models.reldb import (
    MailAccount,
    BaseFeed, basefeed_refresh_task
)

from oneflow.base.utils import RedisExpiringLock
from oneflow.base.utils.dateutils import (now, today, timedelta,
                                          naturaldelta, benchmark)

from gr_import import clean_gri_keys

from mongo import refresh_all_mongo_feeds

# Import this one, so that celery can find it,
# Else it complains about a missing import.
from checks import global_checker_task  # NOQA

LOGGER = logging.getLogger(__name__)


@task(queue='clean')
def clean_obsolete_redis_keys():
    """ Call in turn all redis-related cleaners. """

    start_time = pytime.time()

    if today() <= (config.GR_END_DATE + timedelta(days=1)):
        clean_gri_keys()

    LOGGER.info(u'clean_obsolete_redis_keys(): finished in %s.',
                naturaldelta(pytime.time() - start_time))


@task(queue='high')
def refresh_all_feeds(limit=None, force=False):
    """ Refresh all feeds (RSS/Mail/Twitter…). """

    if config.FEED_FETCH_DISABLED:
        # Do not raise any .retry(), this is a scheduled task.
        LOGGER.warning(u'Feed refresh disabled in configuration.')
        return

    # TODO: WIPE THIS when Mongo → PG migration is complete!
    refresh_all_mongo_feeds.delay()

    # Be sure two refresh operations don't overlap, but don't hold the
    # lock too long if something goes wrong. In production conditions
    # as of 20130812, refreshing all feeds takes only a moment:
    # [2013-08-12 09:07:02,028: INFO/MainProcess] Task
    #       oneflow.core.tasks.refresh_all_feeds succeeded in 1.99886608124s.
    my_lock = RedisExpiringLock('refresh_all_feeds', expire_time=120)

    if not my_lock.acquire():
        if force:
            my_lock.release()
            my_lock.acquire()
            LOGGER.warning(_(u'Forcing all feed refresh…'))

        else:
            # Avoid running this task over and over again in the queue
            # if the previous instance did not yet terminate. Happens
            # when scheduled task runs too quickly.
            LOGGER.warning(u'refresh_all_feeds() is already locked, aborting.')
            return

    # This should bring us a Polymorphic Query to refresh all feeds types.
    feeds = BaseFeed.objects.filter(is_active=True, is_internal=False)

    if limit:
        feeds = feeds.limit(limit)

    with benchmark('refresh_all_feeds()'):

        try:
            count = 0
            mynow = now()

            for feed in feeds:

                if feed.refresh_lock.is_locked():
                    LOGGER.info(u'Feed %s already locked, skipped.', feed)
                    continue

                interval = timedelta(seconds=feed.fetch_interval)

                if feed.date_last_fetch is None:

                    basefeed_refresh_task.delay(feed.id)

                    LOGGER.info(u'Launched immediate refresh of feed %s which '
                                u'has never been refreshed.', feed)

                elif force or feed.date_last_fetch + interval < mynow:

                    how_late = feed.date_last_fetch + interval - mynow
                    how_late = how_late.days * 86400 + how_late.seconds

                    if config.FEED_REFRESH_RANDOMIZE:
                        countdown = randrange(
                            config.FEED_REFRESH_RANDOMIZE_DELAY)

                        basefeed_refresh_task.apply_async((feed.id, force),
                                                          countdown=countdown)

                    else:
                        countdown = 0
                        basefeed_refresh_task.delay(feed.id, force)

                    LOGGER.info(u'%s refresh of feed %s %s (%s late).',
                                u'Scheduled randomized'
                                if countdown else u'Launched',
                                feed,
                                u' in {0}'.format(naturaldelta(countdown))
{'fraction_non_alphanumeric': 0.07489754098360656, 'fraction_numerical': 0.012909836065573771, 'mean_word_length': 2.1466795615731784, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4986832', 'n_tokens_mistral': 2821, 'n_tokens_neox': 2607, 'n_words': 871}