<commit_before># -*- coding: utf-8 -*-
import datetime
import os

from south.db import db
from south.v2 import DataMigration
from django.db import models

class Migration(DataMigration):

    def forwards(self, orm):
        "Write your forwards methods here."
        # Note: Don't use "from appname.models import ModelName".
        # Use orm.ModelName to refer to models in this application,
        # and orm['appname.ModelName'] for models in other applications.

        # Locations
        for location in orm.Location.objects.all():
            newsb = orm.StorageBox()
            newsb.name = location.name
            newsb.description = "converted from Location"
            newsb.status = "dirty"
            if location.type in ('local', 'online') and \
               location.transfer_provider in ('local', ):
                base_dir = location.url.replace('file://', '')
                try:
                    base_dir_stat = os.statvfs(base_dir)
                except OSError:
                    # for running this on a test db
                    print 'Cannot access location %s. OK for testing only' % (
                        location.name,)
                    base_dir_stat = os.statvfs('/')
                disk_size = base_dir_stat.f_frsize * base_dir_stat.f_blocks
                newsb.max_size = disk_size
                newsb.save()
                sb_opt = orm.StorageBoxOption(storage_box=newsb,
                                              key="location",
                                              value=base_dir)
                sb_opt.save()
            else:
                # placeholder storage class. Manual intervention required
                newsb.django_storage_class = \
                    'tardis.tardis_portal.storage.DummyStorage'
                newsb.max_size = 0
                newsb.save()

        # Replicas
        for replica in orm.Replica.objects.all():
            new_dfo = orm.DataFileObject()
            new_dfo.datafile = replica.datafile
            new_dfo.uri = replica.url
            new_dfo.storage_box = orm.StorageBox.objects.get(
                name=replica.location.name)
            new_dfo.save()
            new_dfo.datafile.dataset.storage_boxes.add(new_dfo.storage_box)

        if False:
            files_failed_verification = []
            # verify all files
            for dfo in orm.DataFileObject.objects.all():
                if not dfo.storage_box.django_storage_class == \
                   'tardis.tardis_portal.storage.DummyStorage':
                    if not dfo.verify():
                        files_failed_verification.append(dfo)
            print files_failed_verification or "All files migrated fine"

    def backwards(self, orm):
        "Write your backwards methods here."
        raise RuntimeError("Cannot reverse this migration.")

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'auth.user': {
            'Meta': {'object_name': 'User'},<commit_msg>migration uses less memory and outputs progress information<commit_after># -*- coding: utf-8 -*-
import datetime
import os

from south.db import db
from south.v2 import DataMigration
from django.db import models

class Migration(DataMigration):

    def forwards(self, orm):
        "Write your forwards methods here."
        # Note: Don't use "from appname.models import ModelName".
        # Use orm.ModelName to refer to models in this application,
        # and orm['appname.ModelName'] for models in other applications.

        # Locations
        for location in orm.Location.objects.all():
            newsb = orm.StorageBox()
            newsb.name = location.name
            newsb.description = "converted from Location"
            newsb.status = "dirty"
            if location.type in ('local', 'online') and \
               location.transfer_provider in ('local', ):
                base_dir = location.url.replace('file://', '')
                try:
                    base_dir_stat = os.statvfs(base_dir)
                except OSError:
                    # for running this on a test db
                    print 'Cannot access location %s. OK for testing only' % (
                        location.name,)
                    base_dir_stat = os.statvfs('/')
                disk_size = base_dir_stat.f_frsize * base_dir_stat.f_blocks
                newsb.max_size = disk_size
                newsb.save()
                sb_opt = orm.StorageBoxOption(storage_box=newsb,
                                              key="location",
                                              value=base_dir)
                sb_opt.save()
            else:
                # placeholder storage class. Manual intervention required
                newsb.django_storage_class = \
                    'tardis.tardis_portal.storage.DummyStorage'
                newsb.max_size = 0
                newsb.save()

        # Replicas
        total = float(orm.Replica.objects.all().count())
        counter = 0
        percent = 0
        print 'total replicas: %d ' % int(total)
        import resource
        print 'memory used: %d' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        for replica in orm.Replica.objects.all().iterator():
            new_dfo = orm.DataFileObject()
            new_dfo.datafile = replica.datafile
            new_dfo.uri = replica.url
            new_dfo.storage_box = orm.StorageBox.objects.get(
                name=replica.location.name)
            new_dfo.save()
            new_dfo.datafile.dataset.storage_boxes.add(new_dfo.storage_box)
            counter += 1
            if int(counter/total * 100) > percent:
                percent += 1
                print '{0} % done '.format(percent),
                print 'memory used: %d' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

        if False:
            files_failed_verification = []
            # verify all files
            for dfo in orm.DataFileObject.objects.all():
                if not dfo.storage_box.django_storage_class == \
                   'tardis.tardis_portal.storage.DummyStorage':
                    if not dfo.verify():
                        files_failed_verification.append(dfo)
            print files_failed_verification or "All files migrated fine"

    def backwards(self, orm):
        "Write your backwards methods here."
        raise RuntimeError("Cannot reverse this migration.")

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'auth.user': {
            'Meta': {'object_name': 'User'},
{'fraction_non_alphanumeric': 0.1264461844104242, 'fraction_numerical': 0.0032721748276265047, 'mean_word_length': 1.9089055064581917, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4666802', 'n_tokens_mistral': 2430, 'n_tokens_neox': 2331, 'n_words': 509}