<commit_before>from django.core.management.base import BaseCommand, CommandError
from ...models import Render


class Command(BaseCommand):
    help = 'Deletes output of all expired renders'

    def handle(self, *args, **options):
        for render in Render.objects.expired().iterator():
            try:
                render.delete_output()
            except FileNotFoundError:
                print(f"❌  Render {render.id} already deleted", flush=True)
            else:
                print(f"✅  Render {render.id} deleted", flush=True)
<commit_msg>Batch expired render deletion<commit_after>from django.core.management.base import BaseCommand, CommandError
from ...models import Render


class Command(BaseCommand):
    help = 'Deletes output of all expired renders'

    def handle(self, *args, **options):
        pointer = 0
        batch_size = 1000
        qs = Render.objects.expired()

        while True:
            # Batch because Django just seems to consume loads of memory and lock up
            batch = qs.filter(id__gt=pointer).order_by('id')[:batch_size]
            if not batch:
                break
            for render in batch:
                pointer = render.id
                try:
                    render.delete_output()
                except FileNotFoundError:
                    print(f"❌  Render {render.id} already deleted", flush=True)
                else:
                    print(f"✅  Render {render.id} deleted", flush=True)

{'fraction_non_alphanumeric': 0.09054054054054055, 'fraction_numerical': 0.0033783783783783786, 'mean_word_length': 2.022448979591837, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3228959', 'n_tokens_mistral': 397, 'n_tokens_neox': 377, 'n_words': 116}