<commit_before>from optparse import make_option

from django.core.management.base import BaseCommand, CommandError
from pyelasticsearch import ElasticSearch, ElasticHttpNotFoundError

from ...utils import queryset_iterator, recursive_dict_update, get_all_indexes
from ... import settings as es_settings


class ESCommandError(CommandError):
    pass

class Command(BaseCommand):
    help = ''
    option_list = BaseCommand.option_list + (
        make_option(
            '--indexes',
            action='store',
            dest='indexes',
            default='',
        ),
        make_option(
            '--list',
            action='store_true',
            dest='list',
            default=False,
        ),
        make_option(
            '--no_input',
            action='store_true',
            dest='no_input',
            default=False,
        ),
        make_option(
            '--refresh',
            action='store_true',
            dest='refresh',
            default=False,
        ),
        make_option(
            '--delete',
            action='store_true',
            dest='delete',
            default=False,
        ),
        make_option(
            '--initialize',
            action='store_true',
            dest='initialize',
            default=False,
        ),
        make_option(
            '--rebuild',
            action='store_true',
            dest='rebuild',
            default=False,
        ),
        make_option(
            '--update',
            action='store_true',
            dest='update',
            default=False,
        )
    )
    es = ElasticSearch(es_settings.ES_CONNECTION_URL)

    def handle(self, *args, **options):
        no_input = options.get('no_input')
        # get all available *active* (meaning, in INSTALLED_APPS) ESBaseIndex-derived classes in the project
        self.all_indexes = get_all_indexes(es=self.es)
        # get a copy of the available ES indexes from the active classes
        self.all_index_names = self.all_indexes.keys()

        requested_indexes = None
        if options.get('indexes'):
            requested_indexes = options.get('indexes', '').split(',')
            for index in requested_indexes:
                if index not in self.all_index_names:
                    raise ESCommandError(u"Index '{0}' is not associated with any models through ESBaseIndex-derived classes.".format(index))

        if options.get('list'):
            self.list()
        elif options.get('refresh'):
            self.refresh(requested_indexes, no_input)
        elif options.get('update'):
            self.update(requested_indexes, no_input)
        elif options.get('delete'):
            self.delete(requested_indexes, no_input)
        elif options.get('rebuild'):
            self.rebuild(requested_indexes, no_input)
        elif options.get('initialize'):
            self.initialize(requested_indexes, no_input)

    def list(self):
        print u"Available ES indexes:"
        for index, values in self.all_indexes.iteritems():
            print u" - index '{0}' contains these types:".format(index)
            for value in values:
                print u"  - type '{0}' which indexes the '{1}' model through '{2}'".format(value.get_type_name(), value.get_model().__name__, value.__class__.__name__)

    def refresh(self, indexes=None, no_input=False):
        if no_input or 'yes' == raw_input(u'Are you sure you want to refresh {0} index(es)? [yes/NO]: '.format(u'the ' + u', '.join(indexes) if indexes else '**ALL**')).lower():
            print 'Refreshing...',
            self.es.refresh(indexes)
            print 'complete.'

    def delete(self, indexes=None, no_input=False):
        if no_input or 'yes' == raw_input(u'Are you sure you want to delete {0} index(es)? [yes/NO]: '.format(u'the ' + u', '.join(indexes) if indexes else '**ALL**')).lower():
            print 'Deleting indexes...',
            try:
                if not indexes:
                    self.es.delete_all_indexes()
                else:
                    self.es.delete_index(indexes)
            except ElasticHttpNotFoundError:
                pass
            print 'complete.'

    def initialize(self, indexes=None, no_input=False):
        if no_input or 'yes' == raw_input(u'Are you sure you want to initialize {0} index(es)? [yes/NO]: '.format(u'the ' + u', '.join(indexes) if indexes else '**ALL**')).lower():<commit_msg>Only delete the simple_elasticsearch-managed indexes.<commit_after>from optparse import make_option

from django.core.management.base import BaseCommand, CommandError
from pyelasticsearch import ElasticSearch, ElasticHttpNotFoundError

from ...utils import queryset_iterator, recursive_dict_update, get_all_indexes
from ... import settings as es_settings


class ESCommandError(CommandError):
    pass

class Command(BaseCommand):
    help = ''
    option_list = BaseCommand.option_list + (
        make_option(
            '--indexes',
            action='store',
            dest='indexes',
            default='',
        ),
        make_option(
            '--list',
            action='store_true',
            dest='list',
            default=False,
        ),
        make_option(
            '--no_input',
            action='store_true',
            dest='no_input',
            default=False,
        ),
        make_option(
            '--refresh',
            action='store_true',
            dest='refresh',
            default=False,
        ),
        make_option(
            '--delete',
            action='store_true',
            dest='delete',
            default=False,
        ),
        make_option(
            '--initialize',
            action='store_true',
            dest='initialize',
            default=False,
        ),
        make_option(
            '--rebuild',
            action='store_true',
            dest='rebuild',
            default=False,
        ),
        make_option(
            '--update',
            action='store_true',
            dest='update',
            default=False,
        )
    )
    es = ElasticSearch(es_settings.ES_CONNECTION_URL)

    def handle(self, *args, **options):
        no_input = options.get('no_input')
        # get all available *active* (meaning, in INSTALLED_APPS) ESBaseIndex-derived classes in the project
        self.all_indexes = get_all_indexes(es=self.es)
        # get a copy of the available ES indexes from the active classes
        self.all_index_names = self.all_indexes.keys()

        requested_indexes = None
        if options.get('indexes'):
            requested_indexes = options.get('indexes', '').split(',')
            for index in requested_indexes:
                if index not in self.all_index_names:
                    raise ESCommandError(u"Index '{0}' is not associated with any models through ESBaseIndex-derived classes.".format(index))

        if options.get('list'):
            self.list()
        elif options.get('refresh'):
            self.refresh(requested_indexes, no_input)
        elif options.get('update'):
            self.update(requested_indexes, no_input)
        elif options.get('delete'):
            self.delete(requested_indexes, no_input)
        elif options.get('rebuild'):
            self.rebuild(requested_indexes, no_input)
        elif options.get('initialize'):
            self.initialize(requested_indexes, no_input)

    def list(self):
        print u"Available ES indexes:"
        for index, values in self.all_indexes.iteritems():
            print u" - index '{0}' contains these types:".format(index)
            for value in values:
                print u"  - type '{0}' which indexes the '{1}' model through '{2}'".format(value.get_type_name(), value.get_model().__name__, value.__class__.__name__)

    def refresh(self, indexes=None, no_input=False):
        if no_input or 'yes' == raw_input(u'Are you sure you want to refresh {0} index(es)? [yes/NO]: '.format(u'the ' + u', '.join(indexes) if indexes else '**ALL**')).lower():
            print 'Refreshing...',
            self.es.refresh(indexes)
            print 'complete.'

    def delete(self, indexes=None, no_input=False):
        if no_input or 'yes' == raw_input(u'Are you sure you want to delete {0} index(es)? [yes/NO]: '.format(u'the ' + u', '.join(indexes) if indexes else '**ALL simple_elasticsearch managed**')).lower():
            print 'Deleting indexes...',
            try:
                self.es.delete_index(indexes or self.all_index_names)
            except ElasticHttpNotFoundError:
                pass
            print 'complete.'

    def initialize(self, indexes=None, no_input=False):
        if no_input or 'yes' == raw_input(u'Are you sure you want to initialize {0} index(es)? [yes/NO]: '.format(u'the ' + u', '.join(indexes) if indexes else '**ALL**')).lower():
{'fraction_non_alphanumeric': 0.12064676616915423, 'fraction_numerical': 0.0018091361374943465, 'mean_word_length': 2.0701145435612633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7387102', 'n_tokens_mistral': 2481, 'n_tokens_neox': 2435, 'n_words': 582}