<commit_before>from django.core.management.base import BaseCommand
from django.conf import settings
from apps.rss_feeds.models import Feed
from optparse import make_option
from apps.rss_feeds.tasks import RefreshFeed
import datetime


class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option("-f", "--feed", default=None),
        make_option("-F", "--force", dest="force", action="store_true"),
        make_option('-V', '--verbose', action='store_true',
            dest='verbose', default=False, help='Verbose output.'),
    )

    def handle(self, *args, **options):
        settings.LOG_TO_STREAM = True
        now = datetime.datetime.now()
        
        feeds = Feed.objects.filter(next_scheduled_update__lte=now, active=True).order_by('?')
        
        if options['force']:
            feeds = Feed.objects.all()

        print " ---> Tasking %s feeds..." % feeds.count()

        for f in feeds:
            f.set_next_scheduled_update()
            RefreshFeed.apply_async(args=(f.pk,))
            <commit_msg>Task feeds in a grouped list rather than individually. Saves task creation overhead.<commit_after>from django.core.management.base import BaseCommand
from django.conf import settings
from apps.rss_feeds.models import Feed
from optparse import make_option
from apps.rss_feeds.tasks import RefreshFeed
import datetime


class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option("-f", "--feed", default=None),
        make_option("-F", "--force", dest="force", action="store_true"),
        make_option('-V', '--verbose', action='store_true',
            dest='verbose', default=False, help='Verbose output.'),
    )

    def handle(self, *args, **options):
        settings.LOG_TO_STREAM = True
        now = datetime.datetime.now()
        
        feeds = Feed.objects.filter(next_scheduled_update__lte=now, active=True).order_by('?')
        
        if options['force']:
            feeds = Feed.objects.all().order_by('pk')

        print " ---> Tasking %s feeds..." % feeds.count()
        
        i = 0
        feed_queue = []
        for f in feeds:
            f.set_next_scheduled_update()
            i += 1
            feed_queue.append(f.pk)
            
            if i == 10:
                print feed_queue
                RefreshFeed.apply_async(args=(feed_queue,))
                feed_queue = []
                i = 0
        if feed_queue:
            RefreshFeed.apply_async(args=(feed_queue,))
{'fraction_non_alphanumeric': 0.12514898688915377, 'fraction_numerical': 0.001986491855383393, 'mean_word_length': 2.4165535956580735, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4433739', 'n_tokens_mistral': 743, 'n_tokens_neox': 715, 'n_words': 161}