<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.9.7 on 2016-09-07 18:09
from __future__ import unicode_literals

from django.db import migrations


def DeleteConstraints(table, *keys):
    return migrations.RunSQL(sql='DELETE FROM share_{0}_sources WHERE {0}_id NOT IN (SELECT min(id) FROM share_{0} GROUP BY {1});'.format(table,', '.join(keys)))

def DeleteDuplicates(table, *keys):
    return migrations.RunSQL(sql='DELETE FROM share_{0} WHERE id NOT IN (SELECT min(id) FROM share_{0} GROUP BY {1});'.format(table,', '.join(keys)))


class Migration(migrations.Migration):

    dependencies = [
        ('share', '0036_update_trigger_migrations_20160903_2022'),
    ]

    operations = [
        DeleteConstraints('affiliation', 'person_id', 'entity_id'),
        DeleteDuplicates('affiliation', 'person_id', 'entity_id'),

        DeleteConstraints('association', 'entity_id', 'creative_work_id'),
        DeleteDuplicates('association', 'entity_id', 'creative_work_id'),

        DeleteConstraints('contributor', 'person_id', 'creative_work_id'),
        DeleteDuplicates('contributor', 'person_id', 'creative_work_id'),

        DeleteConstraints('personemail', 'person_id', 'email_id'),
        DeleteDuplicates('personemail', 'person_id', 'email_id'),

        DeleteConstraints('throughawardentities', 'award_id', 'entity_id'),
        DeleteDuplicates('throughawardentities', 'award_id', 'entity_id'),

        DeleteConstraints('throughawards', 'award_id', 'creative_work_id'),
        DeleteDuplicates('throughawards', 'award_id', 'creative_work_id'),

        DeleteConstraints('throughidentifiers', 'person_id', 'identifier_id'),
        DeleteDuplicates('throughidentifiers', 'person_id', 'identifier_id'),

        DeleteConstraints('throughlinks', 'link_id', 'creative_work_id'),
        DeleteDuplicates('throughlinks', 'link_id', 'creative_work_id'),

        DeleteConstraints('throughsubjects', 'subject_id', 'creative_work_id'),
        DeleteDuplicates('throughsubjects', 'subject_id', 'creative_work_id'),

        DeleteConstraints('throughtags', 'tag_id', 'creative_work_id'),
        DeleteDuplicates('throughtags', 'tag_id', 'creative_work_id'),

        DeleteConstraints('throughvenues', 'venue_id', 'creative_work_id'),
        DeleteDuplicates('throughvenues', 'venue_id', 'creative_work_id'),
    ]<commit_msg>Speed up migration<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.9.7 on 2016-09-07 18:09
from __future__ import unicode_literals

from django.db import migrations


def DeleteConstraining(table, *keys):
    return migrations.RunSQL(sql='DELETE FROM share_{0}_sources WHERE {0}_id in (SELECT id FROM share_{0} EXCEPT SELECT min(id) FROM share_{0} GROUP BY {1});'.format(table, ', '.join(keys)))


def DeleteDuplicates(table, *keys):
    return migrations.RunSQL(sql='DELETE FROM share_{0} WHERE id in (SELECT id FROM share_{0} EXCEPT SELECT min(id) FROM share_{0} GROUP BY {1});'.format(table, ', '.join(keys)))


class Migration(migrations.Migration):

    dependencies = [
        ('share', '0036_update_trigger_migrations_20160903_2022'),
    ]

    operations = [
        DeleteConstraining('affiliation', 'person_id', 'entity_id'),
        DeleteDuplicates('affiliation', 'person_id', 'entity_id'),

        DeleteConstraining('association', 'entity_id', 'creative_work_id'),
        DeleteDuplicates('association', 'entity_id', 'creative_work_id'),

        DeleteConstraining('contributor', 'person_id', 'creative_work_id'),
        DeleteDuplicates('contributor', 'person_id', 'creative_work_id'),

        DeleteConstraining('personemail', 'person_id', 'email_id'),
        DeleteDuplicates('personemail', 'person_id', 'email_id'),

        DeleteConstraining('throughawardentities', 'award_id', 'entity_id'),
        DeleteDuplicates('throughawardentities', 'award_id', 'entity_id'),

        DeleteConstraining('throughawards', 'award_id', 'creative_work_id'),
        DeleteDuplicates('throughawards', 'award_id', 'creative_work_id'),

        DeleteConstraining('throughidentifiers', 'person_id', 'identifier_id'),
        DeleteDuplicates('throughidentifiers', 'person_id', 'identifier_id'),

        DeleteConstraining('throughlinks', 'link_id', 'creative_work_id'),
        DeleteDuplicates('throughlinks', 'link_id', 'creative_work_id'),

        DeleteConstraining('throughsubjects', 'subject_id', 'creative_work_id'),
        DeleteDuplicates('throughsubjects', 'subject_id', 'creative_work_id'),

        DeleteConstraining('throughtags', 'tag_id', 'creative_work_id'),
        DeleteDuplicates('throughtags', 'tag_id', 'creative_work_id'),

        DeleteConstraining('throughvenues', 'venue_id', 'creative_work_id'),
        DeleteDuplicates('throughvenues', 'venue_id', 'creative_work_id'),
    ]
{'fraction_non_alphanumeric': 0.14488517745302715, 'fraction_numerical': 0.016701461377870562, 'mean_word_length': 5.205958549222798, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6977957', 'n_tokens_mistral': 1536, 'n_tokens_neox': 1463, 'n_words': 266}