<commit_before># -*- coding: utf-8 -*-

from django.conf import settings
from django.core.management.commands import dumpdata
from django.db import models
from django.db.models import Sum
from django.db.models.signals import post_save, pre_delete

import datetime
import gzip
import os
import time

INDENT = 4
FORMAT = 'json'

PRUNE_CHOICES = (
    ('count', 'Count'),
    ('size', 'Size'),
    ('time', 'Time'),
)


class BackupObject(models.Model):
    """Backup Object

    App to be backed up.  All apps from INSTALLED APPS are added upon `syncdb`.

    """

    app_label = models.CharField(blank=True, max_length=100, unique=True)
    include = models.BooleanField(default=True,
        help_text='Include this app when performing backup?')
    use_natural_keys = models.BooleanField(default=True)
    compress = models.BooleanField(default=True)
    prune_by = models.CharField(max_length=4, choices=PRUNE_CHOICES,
        default='count',
        help_text='What factor leads to archive file deletion?')
    prune_value = models.CharField(blank=True, null=True, max_length='255',
        default='10')
    send_to_admins = models.BooleanField(default=True)
    created = models.DateTimeField(blank=True, auto_now_add=True)
    modified = models.DateTimeField(blank=True, auto_now=True)

    def prune(self):
        if self.prune_by == 'count':
            count = int(self.prune_value)
            if BackupArchive.objects.filter(backup_object=self,
                keep=False).count() > count:
                last_backup = BackupArchive.objects.filter(backup_object=self,
                    keep=False).only('created')[count-1]
                BackupArchive.objects.filter(
                    backup_object=self,
                    keep=False,
                    created__lt=last_backup.created).delete()
        elif self.prune_by == 'size':
            max_size = int(self.prune_value)
            qs = BackupArchive.objects.filter(backup_object=self,
                keep=False).aggregate(Sum('size'))
            size = qs['size__sum']
            ids = list()
            start = 0
            while size > max_size:
                archive = BackupArchive.objects.filter(backup_object=self).only(
                    'id', 'size', 'created').order_by('created')[start]
                ids.append(archive.id)
                start = start + 1
                size = size - archive.size
            BackupArchive.objects.filter(id__in=ids).delete()

    def backup(self, notes=None):
        dt = datetime.datetime.now()
        name = u'%s_%s%s.%s' % (self.app_label, dt.strftime('%Y%j-'),
            int(time.time()), FORMAT)

        if self.compress:
            name = name + u'.gz'

        path = os.path.join(settings.VZ_BACKUP_DIR, name)
        try:
            if self.compress:
                b_file = gzip.open(path, 'wb')
            else:<commit_msg>pep8 on models<commit_after># -*- coding: utf-8 -*-

from django.conf import settings
from django.core.management.commands import dumpdata
from django.db import models
from django.db.models import Sum
from django.db.models.signals import post_save, pre_delete

import datetime
import gzip
import os
import time

INDENT = 4
FORMAT = 'json'

PRUNE_CHOICES = (
    ('count', 'Count'),
    ('size', 'Size'),
    ('time', 'Time'),
)


class BackupObject(models.Model):
    """Backup Object

    App to be backed up.  All apps from INSTALLED APPS are added upon `syncdb`.

    """

    app_label = models.CharField(blank=True, max_length=100, unique=True)
    include = models.BooleanField(default=True,
        help_text='Include this app when performing backup?')
    use_natural_keys = models.BooleanField(default=True)
    compress = models.BooleanField(default=True)
    prune_by = models.CharField(max_length=4, choices=PRUNE_CHOICES,
        default='count',
        help_text='What factor leads to archive file deletion?')
    prune_value = models.CharField(blank=True, null=True, max_length='255',
        default='10')
    send_to_admins = models.BooleanField(default=True)
    created = models.DateTimeField(blank=True, auto_now_add=True)
    modified = models.DateTimeField(blank=True, auto_now=True)

    def prune(self):
        if self.prune_by == 'count':
            count = int(self.prune_value)
            if BackupArchive.objects.filter(backup_object=self,
                keep=False).count() > count:
                last_backup = BackupArchive.objects.filter(backup_object=self,
                    keep=False).only('created')[count-1]
                BackupArchive.objects.filter(
                    backup_object=self,
                    keep=False,
                    created__lt=last_backup.created).delete()
        elif self.prune_by == 'size':
            max_size = int(self.prune_value)
            qs = BackupArchive.objects.filter(backup_object=self,
                keep=False).aggregate(Sum('size'))
            size = qs['size__sum']
            ids = list()
            start = 0
            while size > max_size:
                archive = BackupArchive.objects.filter(
                    backup_object=self).only('id', 'size', 'created').order_by(
                        'created')[start]
                ids.append(archive.id)
                start = start + 1
                size = size - archive.size
            BackupArchive.objects.filter(id__in=ids).delete()

    def backup(self, notes=None):
        dt = datetime.datetime.now()
        name = u'%s_%s%s.%s' % (self.app_label, dt.strftime('%Y%j-'),
            int(time.time()), FORMAT)

        if self.compress:
            name = name + u'.gz'

        path = os.path.join(settings.VZ_BACKUP_DIR, name)
        try:
            if self.compress:
                b_file = gzip.open(path, 'wb')
            else:
{'fraction_non_alphanumeric': 0.1176267678509831, 'fraction_numerical': 0.005001724732666437, 'mean_word_length': 2.3404377880184333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2443311', 'n_tokens_mistral': 1780, 'n_tokens_neox': 1694, 'n_words': 352}