<commit_before>from gevent import monkey; monkey.patch_all()
from corehq.elastic import get_es_new

from pillowtop.es_utils import get_all_expected_es_indices

from cStringIO import StringIO
import traceback
from datetime import datetime
from django.core.mail import mail_admins
from corehq.pillows.user import add_demo_user_to_user_index
import gevent
from django.core.management.base import BaseCommand
from django.core.management import call_command
from django.conf import settings


def get_reindex_commands(alias_name):
    # pillow_command_map is a mapping from es pillows
    # to lists of management commands or functions
    # that should be used to rebuild the index from scratch
    pillow_command_map = {
        'hqdomains': ['ptop_fast_reindex_domains'],
        'hqcases': ['ptop_fast_reindex_cases'],
        'xforms': ['ptop_fast_reindex_xforms'],
        # groupstousers indexing must happen after all users are indexed
        'hqusers': [
            'ptop_fast_reindex_users',
            add_demo_user_to_user_index,
            'ptop_fast_reindex_groupstousers',
            # 'ptop_fast_reindex_unknownusers',  removed until we have a better workflow for this
        ],
        'hqapps': ['ptop_fast_reindex_apps'],
        'hqgroups': ['ptop_fast_reindex_groups'],
        'report_xforms': ['ptop_fast_reindex_reportxforms'],
        'report_cases': ['ptop_fast_reindex_reportcases'],
    }
    return pillow_command_map.get(alias_name, [])


def do_reindex(alias_name):
    print "Starting pillow preindex %s" % alias_name
    reindex_commands = get_reindex_commands(alias_name)
    for reindex_command in reindex_commands:
        if isinstance(reindex_command, basestring):
            call_command(reindex_command, **{'noinput': True, 'bulk': True})
        else:
            reindex_command()
    print "Pillow preindex finished %s" % alias_name


class Command(BaseCommand):
    help = ("Preindex ES pillows. "
            "Only run reindexer if the index doesn't exist.")

    def handle(self, *args, **options):
        runs = []
        all_es_indices = get_all_expected_es_indices()
        es = get_es_new()
        indices_needing_reindex = [info for info in all_es_indices if not es.indices.exists(info.index)]

        if not indices_needing_reindex:
            print 'Nothing needs to be reindexed'
            return

        print "Reindexing:\n\t",
        print '\n\t'.join(map(unicode, indices_needing_reindex))<commit_msg>use new reindexer framework for domain reindexer<commit_after>from gevent import monkey; monkey.patch_all()
from corehq.elastic import get_es_new

from pillowtop.es_utils import get_all_expected_es_indices

from cStringIO import StringIO
import traceback
from datetime import datetime
from django.core.mail import mail_admins
from corehq.pillows.user import add_demo_user_to_user_index
import gevent
from django.core.management.base import BaseCommand
from django.core.management import call_command
from django.conf import settings


def get_reindex_commands(alias_name):
    # pillow_command_map is a mapping from es pillows
    # to lists of management commands or functions
    # that should be used to rebuild the index from scratch
    pillow_command_map = {
        'hqdomains': [('ptop_reindexer_v2', {'index': 'domain'})],
        'hqcases': ['ptop_fast_reindex_cases'],
        'xforms': ['ptop_fast_reindex_xforms'],
        # groupstousers indexing must happen after all users are indexed
        'hqusers': [
            'ptop_fast_reindex_users',
            add_demo_user_to_user_index,
            'ptop_fast_reindex_groupstousers',
            # 'ptop_fast_reindex_unknownusers',  removed until we have a better workflow for this
        ],
        'hqapps': ['ptop_fast_reindex_apps'],
        'hqgroups': ['ptop_fast_reindex_groups'],
        'report_xforms': ['ptop_fast_reindex_reportxforms'],
        'report_cases': ['ptop_fast_reindex_reportcases'],
    }
    return pillow_command_map.get(alias_name, [])


def do_reindex(alias_name):
    print "Starting pillow preindex %s" % alias_name
    reindex_commands = get_reindex_commands(alias_name)
    for reindex_command in reindex_commands:
        if isinstance(reindex_command, basestring):
            call_command(reindex_command, **{'noinput': True, 'bulk': True})
        elif isinstance(reindex_command, (tuple, list)):
            reindex_command, kwargs = reindex_command
            call_command(reindex_command, **kwargs)
        else:
            reindex_command()
    print "Pillow preindex finished %s" % alias_name


class Command(BaseCommand):
    help = ("Preindex ES pillows. "
            "Only run reindexer if the index doesn't exist.")

    def handle(self, *args, **options):
        runs = []
        all_es_indices = get_all_expected_es_indices()
        es = get_es_new()
        indices_needing_reindex = [info for info in all_es_indices if not es.indices.exists(info.index)]

        if not indices_needing_reindex:
            print 'Nothing needs to be reindexed'
            return

        print "Reindexing:\n\t",
        print '\n\t'.join(map(unicode, indices_needing_reindex))
{'fraction_non_alphanumeric': 0.08263821532492725, 'fraction_numerical': 0.00019398642095053346, 'mean_word_length': 3.464069264069264, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3431507', 'n_tokens_mistral': 1585, 'n_tokens_neox': 1554, 'n_words': 392}