<commit_before>import logging

from celery.task import TaskSet
from django.core.management import BaseCommand

from cl.scrapers.tasks import extract_recap_pdf
from cl.search.models import RECAPDocument


logger = logging.getLogger(__name__)


class Command(BaseCommand):
    help = ('Iterate over all of the RECAPDocuments and extract their '
            'content.')

    def handle(self, *args, **options):
        docs = RECAPDocument.objects.filter(
            ocr_status=None
        ).exclude(
            filepath_local='',
        )
        count = docs.count()
        print("There are %s documents to process." % count)

        subtasks = []
        completed = 0
        for pk in docs.values_list('pk', flat=True):
            # Try to extract the contents the easy way, but if that fails,
            # use OCR.
            last_item = (count == completed)
            subtasks.append(extract_recap_pdf.subtask((pk,)))

            # Every n items, send the subtasks to Celery
            if (len(subtasks) >= 50) or last_item:
                msg = ("Sent %s subtasks to celery. We have processed %s "
                       "items so far." % (len(subtasks), completed + 1))<commit_msg>Adds more items to the Celery queue at a time, making it less likely to be held up by a huge PDF.<commit_after>import logging

from celery.task import TaskSet
from django.core.management import BaseCommand

from cl.scrapers.tasks import extract_recap_pdf
from cl.search.models import RECAPDocument


logger = logging.getLogger(__name__)


class Command(BaseCommand):
    help = ('Iterate over all of the RECAPDocuments and extract their '
            'content.')

    def handle(self, *args, **options):
        docs = RECAPDocument.objects.filter(
            ocr_status=None
        ).exclude(
            filepath_local='',
        )
        count = docs.count()
        print("There are %s documents to process." % count)

        subtasks = []
        completed = 0
        for pk in docs.values_list('pk', flat=True):
            # Try to extract the contents the easy way, but if that fails,
            # use OCR.
            last_item = (count == completed)
            subtasks.append(extract_recap_pdf.subtask((pk,)))

            # Every n items, send the subtasks to Celery. The larger this
            # number, the less frequently you must wait while Celery processes a
            # massive PDF.
            enqueue_length = 5000
            if (len(subtasks) >= enqueue_length) or last_item:
                msg = ("Sent %s subtasks to celery. We have sent %s "
                       "items so far." % (len(subtasks), completed + 1))
{'fraction_non_alphanumeric': 0.08974358974358974, 'fraction_numerical': 0.003770739064856712, 'mean_word_length': 2.2041062801932365, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3314680', 'n_tokens_mistral': 748, 'n_tokens_neox': 707, 'n_words': 250}