<commit_before>from django_elasticsearch_dsl.registries import registry

from django_elasticsearch_dsl.management.commands.search_index import Command as DESCommand

CHUNK_SIZE = 100


class Command(DESCommand):
    def _populate(self, models, options):
        for doc in registry.get_documents(models):
            qs = doc().get_queryset()
            self.stdout.write(
                "Indexing {} '{}' objects "
                "with custom chunk_size {}".format(
                    qs.count(), doc._doc_type.model.__name__, CHUNK_SIZE
                )
            )
            doc().update(qs, chunk_size=CHUNK_SIZE)
<commit_msg>Overwrite chunk size in search_index command<commit_after>from django_elasticsearch_dsl.registries import registry

from django_elasticsearch_dsl.management.commands.search_index import Command as DESCommand

CHUNK_SIZE = 100


class Command(DESCommand):
    def _populate(self, models, options):
        for doc in registry.get_documents(models):
            qs = doc().get_queryset()
            chunk_size = getattr(doc._doc_type, 'queryset_pagination', CHUNK_SIZE)
            self.stdout.write(
                "Indexing {} '{}' objects "
                "with custom chunk_size {}".format(
                    qs.count(), doc._doc_type.model.__name__, chunk_size
                )
            )
            doc().update(qs, chunk_size=chunk_size)

{'fraction_non_alphanumeric': 0.09195402298850575, 'fraction_numerical': 0.004310344827586207, 'mean_word_length': 2.4651741293532337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3826904', 'n_tokens_mistral': 400, 'n_tokens_neox': 384, 'n_words': 81}