<commit_before>from optparse import make_option

from django.core.management.base import BaseCommand

from mygpo import migrate
from mygpo.utils import iterate_together, progress
from mygpo.api import models as oldmodels
from mygpo.core import models as newmodels


class Command(BaseCommand):


    option_list = BaseCommand.option_list + (
        make_option('--min-id', action='store', type="int", dest='min_id', default=0, help="Id from which the migration should start."),
        make_option('--max-id', action='store', type="int", dest='max_id', help="Id at which the migration should end."),
    )

    def handle(self, *args, **options):

        min_id = options.get('min_id', 0)
        max_id = options.get('max_id', oldmodels.Episode.objects.order_by('-id')[0].id)

        updated, deleted, created = 0, 0, 0

        oldepisodes = oldmodels.Episode.objects.filter(id__gte=min_id, id__lte=max_id)
        newepisodes = newmodels.Episode.view('core/episodes_by_oldid', startkey=min_id, endkey=max_id, include_docs=True).iterator()
        total = oldepisodes.count()
        compare = lambda o, n: cmp(long(o.id), long(n.oldid))

        for n, (olde, newe) in enumerate(iterate_together(oldepisodes, newepisodes, compare)):

            if (olde != None) and (newe != None):
                podcast = newmodels.Podcast.get(newe.podcast)
                updated += migrate.update_episode(olde, newe)

            elif olde == None:
                deleted += 1
                newe.delete()

            elif newe == None:
                newe = migrate.create_episode(olde)
                created += 1
<commit_msg>bugfixing management command migrate-episodes<commit_after>from optparse import make_option

from django.core.management.base import BaseCommand

from mygpo import migrate
from mygpo.utils import iterate_together, progress
from mygpo.api import models as oldmodels
from mygpo.core import models as newmodels


class Command(BaseCommand):


    option_list = BaseCommand.option_list + (
        make_option('--min-id', action='store', type="int", dest='min_id',
            default=0, help="Id from which the migration should start."),

        make_option('--max-id', action='store', type="int", dest='max_id',
            default=None,  help="Id at which the migration should end."),
    )

    def handle(self, *args, **options):

        min_id = options.get('min_id', 0)
        max_id = options.get('max_id') or oldmodels.Episode.objects.order_by('-id')[0].id

        updated, deleted, created = 0, 0, 0

        oldepisodes = oldmodels.Episode.objects.filter(id__gte=min_id, id__lte=max_id)
        newepisodes = newmodels.Episode.view('core/episodes_by_oldid', startkey=min_id, endkey=max_id, include_docs=True).iterator()
        total = oldepisodes.count()
        compare = lambda o, n: cmp(long(o.id), long(n.oldid))

        for n, (olde, newe) in enumerate(iterate_together(oldepisodes.iterator(), newepisodes, compare)):

            if (olde != None) and (newe != None):
                updated += migrate.update_episode(olde, newe)

            elif olde == None:
                deleted += 1
                #newe.delete()

            elif newe == None:
                try:
                    newe = migrate.create_episode(olde)
                    created += 1
                except oldmodels.Podcast.DoesNotExist:
                    pass

{'fraction_non_alphanumeric': 0.1184792219274978, 'fraction_numerical': 0.004715590922487474, 'mean_word_length': 2.7296703296703297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4147235', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1027, 'n_words': 247}