<commit_before># encoding: utf-8
import sys
from south.db import db
from south.v2 import DataMigration
from django.db import models
from apps.rss_feeds.models import Feed, FeedData
from django.db import transaction

class Migration(DataMigration):

    @transaction.autocommit
    def forwards(self, orm):
        print "FeedData: %s" % FeedData.objects.count()

        feeds = Feed.objects.all().order_by('-average_stories_per_month')
        feed_count = feeds.count()
        i = 0
        for feed in feeds:
            i += 1
            print "%s/%s: %s" % (i, feed_count, feed,)
            sys.stdout.flush()
            data = {
                'feed_tagline': feed.feed_tagline and feed.feed_tagline[:1023],
                'story_count_history': feed.story_count_history,
                'popular_tags': feed.popular_tags and feed.popular_tags[:1023],
                'popular_authors': feed.popular_authors and feed.popular_authors[:2047],
            }
            try:
                sid = transaction.savepoint()
                FeedData.objects.create(feed=feed, **data)
                transaction.savepoint_commit(sid)
            except Exception, e:
                print "!!!!!!!!!!!!!!! Exception: %s" % e
                transaction.savepoint_rollback(sid)
                pass

    def backwards(self, orm):
        print "Start FeedData: %s" % (FeedData.objects.count())
        FeedData.objects.all().delete()
        print "End FeedData: %s" % (FeedData.objects.count())


    models = {
        'rss_feeds.duplicatefeed': {
            'Meta': {'object_name': 'DuplicateFeed'},
            'duplicate_address': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),<commit_msg>Adding sanity checks in FeedData data migration.<commit_after># encoding: utf-8
import sys
from south.db import db
from south.v2 import DataMigration
from django.db import models
from apps.rss_feeds.models import Feed, FeedData
from django.db import transaction

class Migration(DataMigration):

    @transaction.autocommit
    def forwards(self, orm):
        print "FeedData: %s" % FeedData.objects.count()

        f = 0
        while True:
            feeds = Feed.objects.all().order_by('-average_stories_per_month')[f:f+10000]
            f += 10000
            feed_count = feeds.count()
            i = int(f)
            if not feed_count: break
            for feed in feeds:
                i += 1
                print "%s/%s: %s" % (i, feed_count, feed,)
                sys.stdout.flush()
                data = {
                    'feed_tagline': feed.feed_tagline and feed.feed_tagline[:1023],
                    'story_count_history': feed.story_count_history,
                    'popular_tags': feed.popular_tags and feed.popular_tags[:1023],
                    'popular_authors': feed.popular_authors and feed.popular_authors[:2047],
                }
                try:
                    sid = transaction.savepoint()
                    FeedData.objects.create(feed=feed, **data)
                    transaction.savepoint_commit(sid)
                except Exception, e:
                    print "!!!!!!!!!!!!!!! Exception: %s" % e
                    transaction.savepoint_rollback(sid)
                    pass

    def backwards(self, orm):
        print "Start FeedData: %s" % (FeedData.objects.count())
        FeedData.objects.all().delete()
        print "End FeedData: %s" % (FeedData.objects.count())


    models = {
        'rss_feeds.duplicatefeed': {
            'Meta': {'object_name': 'DuplicateFeed'},
            'duplicate_address': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
{'fraction_non_alphanumeric': 0.12375033774655499, 'fraction_numerical': 0.012969467711429343, 'mean_word_length': 2.0696517412935322, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2886049', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1021, 'n_words': 235}