<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations
from django.db.models import Max

BATCH_SIZE = 5000
MSG_BATCH = BATCH_SIZE * 5


def update_msg_purge_status(Broadcast, Msg):

    from django.utils import timezone
    from datetime import timedelta

    # 90 days ago
    purge_date = timezone.now() - timedelta(days=90)

    if getattr(Msg, 'objects', None):
        msgs = Msg.objects
    else:
        msgs = Msg.current_messages

    max_pk = Broadcast.objects.aggregate(Max('pk'))['pk__max']
    if max_pk is not None:
        print "Populating broadcasts purged field.."
        for offset in range(0, max_pk+1, BATCH_SIZE):
            print 'On %d of %d' % (offset, max_pk)

            # determine which broadcasts are old
            broadcasts = Broadcast.objects.filter(pk__gte=offset,
                                                  pk__lt=offset+BATCH_SIZE,
                                                  created_on__lt=purge_date,
                                                  purged__isnull=True)

            # set our old broadcast purge
            broadcasts.update(purged=False)

            # store the broadcasts we purged
            purged_broadcasts = [b.id for b in broadcasts]

            # all the related messages for those broadcasts
            max_msg_pk = msgs.aggregate(Max('pk'))['pk__max']
            for msg_offset in range(0, max_msg_pk+1, MSG_BATCH*5):
                # msg part of a purged broadcast
                msgs.filter(pk__gte=msg_offset,
                            pk__lt=msg_offset+MSG_BATCH,
                            purged__isnull=True,
                            broadcast_id__in=purged_broadcasts).update(purged=True)

            # any other unset broadcasts are considered not purged
            Broadcast.objects.filter(pk__gte=offset,
                                     pk__lt=offset+BATCH_SIZE,
                                     purged__isnull=True).update(purged=False)

    max_pk = msgs.aggregate(Max('pk'))['pk__max']
    if max_pk is not None:
        print "Populating messages purged field.."
        for offset in range(0, max_pk+1, BATCH_SIZE):
            print 'On %d of %d' % (offset, max_pk)

            # all remaining messages
            msgs.filter(pk__gte=offset,
                        pk__lt=offset+BATCH_SIZE,
                        purged__isnull=True).update(purged=False)


def update_purge(apps, schema_editor):

    # we want to be non-atomic
    if schema_editor.connection.in_atomic_block:
            schema_editor.atomic.__exit__(None, None, None)
    update_msg_purge_status(apps.get_model('msgs', 'Broadcast'), apps.get_model('msgs', 'Msg'))


class Migration(migrations.Migration):<commit_msg>Add optional parameters to purge migration<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations
from django.db.models import Max
from django.utils import timezone
from datetime import timedelta


def update_msg_purge_status(Broadcast, Msg, update_broadcasts=True, batch_size=5000, msg_start=0):

    msg_batch = batch_size * 5

    # 90 days ago
    purge_date = timezone.now() - timedelta(days=90)

    if getattr(Msg, 'objects', None):
        msgs = Msg.objects
    else:
        msgs = Msg.current_messages

    if update_broadcasts:
        max_pk = Broadcast.objects.aggregate(Max('pk'))['pk__max']
        if max_pk is not None:
            print "Populating broadcasts purged field.."
            for offset in range(0, max_pk+1, batch_size):
                print 'Broadcast %d of %d' % (offset, max_pk)

                # determine which broadcasts are old
                broadcasts = Broadcast.objects.filter(pk__gte=offset,
                                                      pk__lt=offset+batch_size,
                                                      created_on__lt=purge_date,
                                                      purged__isnull=True)

                # set our old broadcast purge
                broadcasts.update(purged=False)

                # store the broadcasts we purged
                purged_broadcasts = [b.id for b in broadcasts]

                # all the related messages for those broadcasts
                max_msg_pk = msgs.aggregate(Max('pk'))['pk__max']
                for msg_offset in range(msg_start, max_msg_pk+1, msg_batch):
                    # msg part of a purged broadcast
                    msgs.filter(pk__gte=msg_offset,
                                pk__lt=msg_offset+msg_batch,
                                purged__isnull=True,
                                broadcast_id__in=purged_broadcasts).update(purged=True)

                # any other unset broadcasts are considered not purged
                Broadcast.objects.filter(pk__gte=offset,
                pk__lt=offset+batch_size,
                                         purged__isnull=True).update(purged=False)

    max_pk = msgs.aggregate(Max('pk'))['pk__max']
    if max_pk is not None:
        print "Populating messages purged field.. (start %d)" % msg_start
        for offset in range(msg_start, max_pk+1, batch_size):
            print 'Msg %d of %d' % (offset, max_pk)

            # all remaining messages
            msgs.filter(pk__gte=offset,
                        pk__lt=offset+batch_size,
                        purged__isnull=True).update(purged=False)


def update_purge(apps, schema_editor):

    # we want to be non-atomic
    if schema_editor.connection.in_atomic_block:
            schema_editor.atomic.__exit__(None, None, None)
    update_msg_purge_status(apps.get_model('msgs', 'Broadcast'), apps.get_model('msgs', 'Msg'))


class Migration(migrations.Migration):
{'fraction_non_alphanumeric': 0.08278375564040263, 'fraction_numerical': 0.0055536272127733426, 'mean_word_length': 1.9018126888217524, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5519684', 'n_tokens_mistral': 1692, 'n_tokens_neox': 1586, 'n_words': 378}