<commit_before>import datetime
import os
import shutil
import time
from celery.task import Task
from utils import log as logging
from utils import s3_utils as s3
from django.conf import settings

class TaskFeeds(Task):
    name = 'task-feeds'

    def run(self, **kwargs):
        from apps.rss_feeds.models import Feed        
        settings.LOG_TO_STREAM = True
        now = datetime.datetime.utcnow()
        
        # Active feeds
        feeds = Feed.objects.filter(
            next_scheduled_update__lte=now,
            active=True,
            active_premium_subscribers__gte=10
        ).order_by('?')
        active_count = feeds.count()
        
        # Regular feeds
        feeds = Feed.objects.filter(
            next_scheduled_update__lte=now,
            active=True,
            active_premium_subscribers__gte=0
        ).order_by('?')[:1000]
        active_count = feeds.count()
        <commit_msg>Only 5k feeds for minute.<commit_after>import datetime
import os
import shutil
import time
from celery.task import Task
from utils import log as logging
from utils import s3_utils as s3
from django.conf import settings

class TaskFeeds(Task):
    name = 'task-feeds'

    def run(self, **kwargs):
        from apps.rss_feeds.models import Feed        
        settings.LOG_TO_STREAM = True
        now = datetime.datetime.utcnow()
        
        # Active feeds
        feeds = Feed.objects.filter(
            next_scheduled_update__lte=now,
            active=True,
            active_premium_subscribers__gte=10
        ).order_by('?')[:5000]
        active_count = feeds.count()
        
        # Regular feeds
        feeds = Feed.objects.filter(
            next_scheduled_update__lte=now,
            active=True,
            active_premium_subscribers__gte=0
        ).order_by('?')[:1000]
        active_count = feeds.count()
        
{'fraction_non_alphanumeric': 0.07910208444681988, 'fraction_numerical': 0.012292891501870658, 'mean_word_length': 2.1728813559322036, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4975087', 'n_tokens_mistral': 574, 'n_tokens_neox': 546, 'n_words': 130}