<commit_before>from __future__ import unicode_literals

from datetime import datetime

from django.core.management.base import BaseCommand, CommandError
from osf_models.scripts.load_blacklist_guids import \
    main as load_blacklist_guids
from osf_models.scripts.load_guids import main as load_guids
from osf_models.scripts.migrate_nodelogs import main as migrate_nodelogs
from osf_models.scripts.migrate_nodes import (build_pk_caches,
                                              save_bare_embargos,
                                              save_bare_retractions,
                                              save_bare_nodes,
                                              save_bare_system_tags,
                                              save_bare_tags, save_bare_users,
                                              set_node_foreign_keys_on_nodes,
                                              set_node_many_to_many_on_nodes,
                                              set_node_many_to_many_on_users,
                                              set_system_tag_many_to_many_on_users,
                                              set_tag_many_to_many_on_nodes,
                                              set_user_foreign_keys_on_nodes,
                                              set_user_foreign_keys_on_users,
                                              set_user_many_to_many_on_nodes,
                                              set_user_many_to_many_on_users, set_retraction_foreign_keys_on_nodes,
                                              set_embargo_foreign_keys_on_nodes, merge_duplicate_users)
from osf_models.scripts.verify_guids import main as verify_guids
from osf_models.scripts.verify_nodelogs import main as verify_nodelogs
from osf_models.scripts.verify_nodes import main as verify_nodes
from website.app import init_app


class Command(BaseCommand):
    help = 'Migrates data from tokumx to postgres'

    def handle(self, *args, **options):
        print 'Initializing Flask App...'
        init_app()
        start = datetime.now()
        #
        # load_guids()
        # print 'Loaded Guids in {} seconds...'.format((datetime.now() - start
        #                                               ).total_seconds())
        # snap = datetime.now()
        # load_blacklist_guids()
        # print 'Loaded Blacklist in {} seconds...'.format((datetime.now() - snap
        #                                                   ).total_seconds())
        # save_bare_nodes()
        merge_duplicate_users()
        save_bare_users()
        save_bare_tags()
        save_bare_system_tags()

        global modm_to_django
        modm_to_django = build_pk_caches()
        print 'Cached {} MODM to django mappings...'.format(len(
            modm_to_django.keys()))

        save_bare_embargos()
        save_bare_retractions()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        global modm_to_django
        modm_to_django = build_pk_caches()
        print 'Cached {} MODM to django mappings...'.format(len(
            modm_to_django.keys()))

        # fk
        set_node_foreign_keys_on_nodes()
        set_retraction_foreign_keys_on_nodes()
        set_embargo_foreign_keys_on_nodes()
        set_user_foreign_keys_on_nodes()
        set_user_foreign_keys_on_users()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        global modm_to_django
        modm_to_django = build_pk_caches()
        print 'Cached {} MODM to django mappings...'.format(len(
            modm_to_django.keys()))

        # m2m
        set_node_many_to_many_on_nodes()
        set_user_many_to_many_on_nodes()
        set_node_many_to_many_on_users()
        set_user_many_to_many_on_users()
        set_system_tag_many_to_many_on_users()
        set_tag_many_to_many_on_nodes()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        global modm_to_django
        modm_to_django = build_pk_caches()
        print 'Cached {} MODM to django mappings...'.format(len(
            modm_to_django.keys()))

        # verify
        verify_guids()
        verify_nodes()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        # nodelogs
        migrate_nodelogs()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        # verify nodelogs
        verify_nodelogs()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        print 'Finished in {} seconds...'.format((datetime.now() - start
                                                  ).total_seconds())
<commit_msg>Add stuff back into migrations<commit_after>from __future__ import unicode_literals

from datetime import datetime

from django.core.management.base import BaseCommand, CommandError
from osf_models.scripts.load_blacklist_guids import \
    main as load_blacklist_guids
from osf_models.scripts.load_guids import main as load_guids
from osf_models.scripts.migrate_nodelogs import main as migrate_nodelogs
from osf_models.scripts.migrate_nodes import (build_pk_caches,
                                              save_bare_embargos,
                                              save_bare_retractions,
                                              save_bare_nodes,
                                              save_bare_system_tags,
                                              save_bare_tags, save_bare_users,
                                              set_node_foreign_keys_on_nodes,
                                              set_node_many_to_many_on_nodes,
                                              set_node_many_to_many_on_users,
                                              set_system_tag_many_to_many_on_users,
                                              set_tag_many_to_many_on_nodes,
                                              set_user_foreign_keys_on_nodes,
                                              set_user_foreign_keys_on_users,
                                              set_user_many_to_many_on_nodes,
                                              set_user_many_to_many_on_users, set_retraction_foreign_keys_on_nodes,
                                              set_embargo_foreign_keys_on_nodes, merge_duplicate_users)
from osf_models.scripts.verify_guids import main as verify_guids
from osf_models.scripts.verify_nodelogs import main as verify_nodelogs
from osf_models.scripts.verify_nodes import main as verify_nodes
from website.app import init_app


class Command(BaseCommand):
    help = 'Migrates data from tokumx to postgres'

    def handle(self, *args, **options):
        print 'Initializing Flask App...'
        init_app()
        start = datetime.now()

        load_guids()
        print 'Loaded Guids in {} seconds...'.format((datetime.now() - start
                                                      ).total_seconds())
        snap = datetime.now()
        load_blacklist_guids()
        print 'Loaded Blacklist in {} seconds...'.format((datetime.now() - snap
                                                          ).total_seconds())
        save_bare_nodes()
        merge_duplicate_users()
        save_bare_users()
        save_bare_tags()
        save_bare_system_tags()

        global modm_to_django
        modm_to_django = build_pk_caches()
        print 'Cached {} MODM to django mappings...'.format(len(
            modm_to_django.keys()))

        save_bare_embargos()
        save_bare_retractions()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        global modm_to_django
        modm_to_django = build_pk_caches()
        print 'Cached {} MODM to django mappings...'.format(len(
            modm_to_django.keys()))

        # fk
        set_node_foreign_keys_on_nodes()
        set_retraction_foreign_keys_on_nodes()
        set_embargo_foreign_keys_on_nodes()
        set_user_foreign_keys_on_nodes()
        set_user_foreign_keys_on_users()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        global modm_to_django
        modm_to_django = build_pk_caches()
        print 'Cached {} MODM to django mappings...'.format(len(
            modm_to_django.keys()))

        # m2m
        set_node_many_to_many_on_nodes()
        set_user_many_to_many_on_nodes()
        set_node_many_to_many_on_users()
        set_user_many_to_many_on_users()
        set_system_tag_many_to_many_on_users()
        set_tag_many_to_many_on_nodes()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        global modm_to_django
        modm_to_django = build_pk_caches()
        print 'Cached {} MODM to django mappings...'.format(len(
            modm_to_django.keys()))

        # verify
        verify_guids()
        verify_nodes()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        # nodelogs
        migrate_nodelogs()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        # verify nodelogs
        verify_nodelogs()
        print '\a\a\a\a\a\a\a\a\a\a\a\a'

        print 'Finished in {} seconds...'.format((datetime.now() - start
                                                  ).total_seconds())

{'fraction_non_alphanumeric': 0.07691443970357063, 'fraction_numerical': 0.00022456770716370987, 'mean_word_length': 1.567598731622946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 72, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5825668', 'n_tokens_mistral': 2613, 'n_tokens_neox': 2528, 'n_words': 410}