<commit_before>from optparse import make_option
import time

from django.conf import settings
from django.core.management.base import BaseCommand

from lily.search.analyzers import get_analyzers
from lily.search.connections_utils import get_es_client, get_index_name
from lily.search.indexing import index_objects
from lily.search.scan_search import ModelMappings


class Command(BaseCommand):
    help = """Index current model instances into Elasticsearch. It does this by
creating a new index, then changing the alias to point to the new index.
(afterwards removing the old index). It uses 1 index per type.

It uses custom index implementation for fast and synchronous indexing.

There are basically two ways to use this command, the first to index all
configured mappings:

    index

The second way is to target a specific model:

    index -t contact

or with other names:

    index -t lily.contacts.models.Contact
    index -t contacts_contact

It is possible to specify multiple models, using comma separation."""

    option_list = BaseCommand.option_list + (
        make_option('-t', '--target',
                    action='store',
                    dest='target',
                    default='',
                    help='Choose specific model targets, comma separated (no added space after comma).'
                    ),
        make_option('-q', '--queries',
                    action='store_true',
                    dest='queries',
                    help='Show the queries that were executed during the command.'
                    ),
        make_option('-l', '--list',
                    action='store_true',
                    dest='list',
                    help='List available models to target.'
                    ),
        make_option('-f', '--force',
                    action='store_true',
                    dest='force',
                    help='Force the creation of the new index, removing the old one (leftovers).'
                    ),
    )

    def handle(self, *args, **options):
        es = get_es_client()

        if args:
            self.stdout.write('Aborting, unexpected arguments %s' % list(args))
            return

        if options['list']:
            self.stdout.write('Possible models to index:\n')
            for mapping in ModelMappings.get_model_mappings().values():
                self.stdout.write(mapping.get_mapping_type_name())
            return

        target = options['target']
        if target:
            targets = target.split(',')
        else:
            targets = []  # (meaning all)
        has_targets = targets != []

        self.stdout.write('Please remember that HelloLily needs to be in maintenance mode. \n\n')

        if has_targets:
            # Do a quick run to check if all targets are valid models.
            check_targets = list(targets)  # make a copy
            for target in check_targets:
                for mapping in ModelMappings.get_model_mappings().values():
                    if self.model_targetted(mapping, [target]):
                        check_targets.remove(target)
                        break
            if check_targets:
                self.stdout.write('Aborting, following targets not recognized: %s' % check_targets)
                return

        for mapping in ModelMappings.get_model_mappings().values():
            model_name = mapping.get_mapping_type_name()
            main_index_base = settings.ES_INDEXES['default']
            main_index = get_index_name(main_index_base, mapping)

            # Skip this model if there are specific targets and not specified.
            if has_targets and not self.model_targetted(mapping, targets):
                continue

            self.stdout.write('==> %s' % model_name)

            # Check if we currently have an index for this mapping.
            old_index = None
            aliases = es.indices.get_aliases(name=main_index)
            for key, value in aliases.iteritems():
                if value['aliases']:
                    old_index = key
                    self.stdout.write('Current index "%s"' % key)

            # Check any indices with no alias (leftovers from failed indexing).
            # Or it could be that it is still in progress,
            aliases = es.indices.get_aliases()
            for key, value in aliases.iteritems():
                if not key.endswith(model_name):
                    # Not the model we are looking after.
                    continue
                if key == main_index:
                    # This is an auto created index. Will be removed at end of command.
                    continue
                if not value['aliases']:
                    if options.get('force', ''):
                        self.stdout.write('Removing leftover "%s"' % key)
                        es.indices.delete(key)
                    else:
                        raise Exception('Found leftover %s, proceed with -f to remove.'
                                        ' Make sure indexing this model is not already running!' % key)

            # Create new index.
            index_settings = {
                'mappings': {
                    model_name: mapping.get_mapping()
                },
                'settings': {
                    'analysis': get_analyzers()['analysis'],
                    'number_of_shards': 1,
                }
            }
            temp_index_base = 'index_%s' % (int(time.time()))
            temp_index = get_index_name(temp_index_base, mapping)

            self.stdout.write('Creating new index "%s"' % temp_index)
            es.indices.create(temp_index, body=index_settings)<commit_msg>Revert "Fixed KeyError in index command"<commit_after>from optparse import make_option
import time

from django.conf import settings
from django.core.management.base import BaseCommand

from lily.search.analyzers import get_analyzers
from lily.search.connections_utils import get_es_client, get_index_name
from lily.search.indexing import index_objects
from lily.search.scan_search import ModelMappings


class Command(BaseCommand):
    help = """Index current model instances into Elasticsearch. It does this by
creating a new index, then changing the alias to point to the new index.
(afterwards removing the old index). It uses 1 index per type.

It uses custom index implementation for fast and synchronous indexing.

There are basically two ways to use this command, the first to index all
configured mappings:

    index

The second way is to target a specific model:

    index -t contact

or with other names:

    index -t lily.contacts.models.Contact
    index -t contacts_contact

It is possible to specify multiple models, using comma separation."""

    option_list = BaseCommand.option_list + (
        make_option('-t', '--target',
                    action='store',
                    dest='target',
                    default='',
                    help='Choose specific model targets, comma separated (no added space after comma).'
                    ),
        make_option('-q', '--queries',
                    action='store_true',
                    dest='queries',
                    help='Show the queries that were executed during the command.'
                    ),
        make_option('-l', '--list',
                    action='store_true',
                    dest='list',
                    help='List available models to target.'
                    ),
        make_option('-f', '--force',
                    action='store_true',
                    dest='force',
                    help='Force the creation of the new index, removing the old one (leftovers).'
                    ),
    )

    def handle(self, *args, **options):
        es = get_es_client()

        if args:
            self.stdout.write('Aborting, unexpected arguments %s' % list(args))
            return

        if options['list']:
            self.stdout.write('Possible models to index:\n')
            for mapping in ModelMappings.get_model_mappings().values():
                self.stdout.write(mapping.get_mapping_type_name())
            return

        target = options['target']
        if target:
            targets = target.split(',')
        else:
            targets = []  # (meaning all)
        has_targets = targets != []

        self.stdout.write('Please remember that HelloLily needs to be in maintenance mode. \n\n')

        if has_targets:
            # Do a quick run to check if all targets are valid models.
            check_targets = list(targets)  # make a copy
            for target in check_targets:
                for mapping in ModelMappings.get_model_mappings().values():
                    if self.model_targetted(mapping, [target]):
                        check_targets.remove(target)
                        break
            if check_targets:
                self.stdout.write('Aborting, following targets not recognized: %s' % check_targets)
                return

        for mapping in ModelMappings.get_model_mappings().values():
            model_name = mapping.get_mapping_type_name()
            main_index_base = settings.ES_INDEXES['default']
            main_index = get_index_name(main_index_base, mapping)

            # Skip this model if there are specific targets and not specified.
            if has_targets and not self.model_targetted(mapping, targets):
                continue

            self.stdout.write('==> %s' % model_name)

            # Check if we currently have an index for this mapping.
            old_index = None
            aliases = es.indices.get_aliases(name=main_index)
            for key, value in aliases.iteritems():
                if value['aliases']:
                    old_index = key
                    self.stdout.write('Current index "%s"' % key)

            # Check any indices with no alias (leftovers from failed indexing).
            # Or it could be that it is still in progress,
            aliases = es.indices.get_aliases()
            for key, value in aliases.iteritems():
                if not key.endswith(model_name):
                    # Not the model we are looking after.
                    continue
                if key == main_index:
                    # This is an auto created index. Will be removed at end of command.
                    continue
                if not value['aliases']:
                    if options['force']:
                        self.stdout.write('Removing leftover "%s"' % key)
                        es.indices.delete(key)
                    else:
                        raise Exception('Found leftover %s, proceed with -f to remove.'
                                        ' Make sure indexing this model is not already running!' % key)

            # Create new index.
            index_settings = {
                'mappings': {
                    model_name: mapping.get_mapping()
                },
                'settings': {
                    'analysis': get_analyzers()['analysis'],
                    'number_of_shards': 1,
                }
            }
            temp_index_base = 'index_%s' % (int(time.time()))
            temp_index = get_index_name(temp_index_base, mapping)

            self.stdout.write('Creating new index "%s"' % temp_index)
            es.indices.create(temp_index, body=index_settings)
{'fraction_non_alphanumeric': 0.08430666549574076, 'fraction_numerical': 0.0003512777728989198, 'mean_word_length': 1.706273764258555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2343152', 'n_tokens_mistral': 2912, 'n_tokens_neox': 2702, 'n_words': 892}