<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from corehq.sql_db.operations import HqRunPython


RENAMES = (
    ("applications-to-elasticsearch", "ApplicationToElasticsearchPillow-hqapps_2016-07-11_2128"),
    ("all-cases-to-elasticsearch", "CaseToElasticsearchPillow-hqcases_2016-03-04"),
    ("all-xforms-to-elasticsearch", "XFormToElasticsearchPillow-xforms_2016-07-07"),
    ("case-search-to-elasticsearch", "CaseSearchToElasticsearchPillow-case_search_2016-03-15"),
    ("GroupPillow", "GroupPillow-hqgroups_20150403_1501"),
    ("GroupToUserPillow", "GroupToUserPillow-hqusers_2016-07-19"),
    ("KafkaDomainPillow", "KafkaDomainPillow-hqdomains_2016-08-08"),
    ("ledger-to-elasticsearch", "LedgerToElasticsearchPillow-ledgers_2016-03-15"),
    ("report-cases-to-elasticsearch", "ReportCaseToElasticsearchPillow-report_cases_czei39du507m9mmpqk3y01x72a3ux4p0"),
    ("report-xforms-to-elasticsearch", "ReportXFormToElasticsearchPillow-report_xforms_20160707_2322"),
    ("sql-sms-to-es", "SqlSMSPillow-smslogs_708c77f8e5fe00286fa5791e9fa7d45f"),
    ("UnknownUsersPillow", "UnknownUsersPillow-hqusers_2016-07-19"),
    ("UserPillow", "UserPillow-hqusers_2016-07-19"),
)


def copy_checkpoints(apps, schema_editor):
    from pillowtop import get_all_pillow_instances
    DjangoPillowCheckpoint = apps.get_model('pillowtop', 'DjangoPillowCheckpoint')
    current_checkoint_ids = set(
        [pillow_instance.checkpoint.checkpoint_id for pillow_instance in get_all_pillow_instances()]
    )

    for old_id, new_id in RENAMES:
        try:
            checkpoint = DjangoPillowCheckpoint.objects.get(checkpoint_id=old_id)
            # since checkpoint_id is the primary key, this should make a new model
            # which is good in case we need to rollback
            checkpoint.checkpoint_id = new_id
            checkpoint.save()
            assert new_id in current_checkoint_ids
        except DjangoPillowCheckpoint.DoesNotExist:
            print 'warning: legacy pillow checkpoint with ID {} not found'.format(old_id)
        except AssertionError:
            print 'warning: no current pillow found with checkpoint ID {}'.format(new_id)


def delete_checkpoints(apps, schema_editor):<commit_msg>Silence pillow checkpoint warnings in tests<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.db import models, migrations
from corehq.sql_db.operations import HqRunPython


RENAMES = (
    ("applications-to-elasticsearch", "ApplicationToElasticsearchPillow-hqapps_2016-07-11_2128"),
    ("all-cases-to-elasticsearch", "CaseToElasticsearchPillow-hqcases_2016-03-04"),
    ("all-xforms-to-elasticsearch", "XFormToElasticsearchPillow-xforms_2016-07-07"),
    ("case-search-to-elasticsearch", "CaseSearchToElasticsearchPillow-case_search_2016-03-15"),
    ("GroupPillow", "GroupPillow-hqgroups_20150403_1501"),
    ("GroupToUserPillow", "GroupToUserPillow-hqusers_2016-07-19"),
    ("KafkaDomainPillow", "KafkaDomainPillow-hqdomains_2016-08-08"),
    ("ledger-to-elasticsearch", "LedgerToElasticsearchPillow-ledgers_2016-03-15"),
    ("report-cases-to-elasticsearch", "ReportCaseToElasticsearchPillow-report_cases_czei39du507m9mmpqk3y01x72a3ux4p0"),
    ("report-xforms-to-elasticsearch", "ReportXFormToElasticsearchPillow-report_xforms_20160707_2322"),
    ("sql-sms-to-es", "SqlSMSPillow-smslogs_708c77f8e5fe00286fa5791e9fa7d45f"),
    ("UnknownUsersPillow", "UnknownUsersPillow-hqusers_2016-07-19"),
    ("UserPillow", "UserPillow-hqusers_2016-07-19"),
)


def copy_checkpoints(apps, schema_editor):
    from pillowtop import get_all_pillow_instances
    DjangoPillowCheckpoint = apps.get_model('pillowtop', 'DjangoPillowCheckpoint')
    current_checkoint_ids = set(
        [pillow_instance.checkpoint.checkpoint_id for pillow_instance in get_all_pillow_instances()]
    )

    for old_id, new_id in RENAMES:
        try:
            checkpoint = DjangoPillowCheckpoint.objects.get(checkpoint_id=old_id)
            # since checkpoint_id is the primary key, this should make a new model
            # which is good in case we need to rollback
            checkpoint.checkpoint_id = new_id
            checkpoint.save()
        except DjangoPillowCheckpoint.DoesNotExist:
            if not settings.UNIT_TESTING:
                print 'warning: legacy pillow checkpoint with ID {} not found'.format(old_id)
        if new_id not in current_checkoint_ids and not settings.UNIT_TESTING:
            print 'warning: no current pillow found with checkpoint ID {}'.format(new_id)


def delete_checkpoints(apps, schema_editor):
{'fraction_non_alphanumeric': 0.10245464247598719, 'fraction_numerical': 0.0576307363927428, 'mean_word_length': 4.525943396226415, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5832586', 'n_tokens_mistral': 1746, 'n_tokens_neox': 1591, 'n_words': 257}