<commit_before>from corehq.util.elastic import delete_production_es_index
from pillowtop.es_utils import set_index_reindex_settings, \
    set_index_normal_settings, get_index_info_from_pillow, initialize_mapping_if_necessary
from pillowtop.pillow.interface import PillowRuntimeContext


class PillowReindexer(object):

    def __init__(self, pillow, change_provider):
        self.pillow = pillow
        self.change_provider = change_provider

    def clean_index(self):
        """
        Cleans the index.

        This can be called prior to reindex to ensure starting from a clean slate.
        Should be overridden on a case-by-case basis by subclasses.
        """
        pass

    def reindex(self, start_from=None):
        reindexer_context = PillowRuntimeContext(do_set_checkpoint=False)
        for change in self.change_provider.iter_changes(start_from=start_from):
            self.pillow.processor(change, reindexer_context)


class ElasticPillowReindexer(PillowReindexer):

    def __init__(self, pillow, change_provider, elasticsearch, index_info):
        super(ElasticPillowReindexer, self).__init__(pillow, change_provider)
        self.es = elasticsearch
        self.index_info = index_info

    def clean_index(self):
        delete_production_es_index(self.index_info.index)

    def reindex(self, start_from=None):
        if not start_from:
            # when not resuming force delete and create the index
            self._prepare_index_for_reindex()

        super(ElasticPillowReindexer, self).reindex(start_from)

        self._prepare_index_for_usage()

    def _prepare_index_for_reindex(self):
        if not self.es.indices.exists(self.index_info.index):
            self.es.indices.create(index=self.index_info.index, body=self.index_info.meta)
        initialize_mapping_if_necessary(self.es, self.index_info)
        set_index_reindex_settings(self.es, self.index_info.index)

    def _prepare_index_for_usage(self):
        set_index_normal_settings(self.es, self.index_info.index)
        self.es.indices.refresh(self.index_info.index)


def get_default_reindexer_for_elastic_pillow(pillow, change_provider):
    return ElasticPillowReindexer(
        pillow=pillow,
        change_provider=change_provider,
        elasticsearch=pillow.get_es_new(),
        index_info=get_index_info_from_pillow(pillow),
    )
<commit_msg>only delete index if it exists, use library function<commit_after>from pillowtop.es_utils import set_index_reindex_settings, \
    set_index_normal_settings, get_index_info_from_pillow, initialize_mapping_if_necessary
from pillowtop.pillow.interface import PillowRuntimeContext


class PillowReindexer(object):

    def __init__(self, pillow, change_provider):
        self.pillow = pillow
        self.change_provider = change_provider

    def clean_index(self):
        """
        Cleans the index.

        This can be called prior to reindex to ensure starting from a clean slate.
        Should be overridden on a case-by-case basis by subclasses.
        """
        pass

    def reindex(self, start_from=None):
        reindexer_context = PillowRuntimeContext(do_set_checkpoint=False)
        for change in self.change_provider.iter_changes(start_from=start_from):
            self.pillow.processor(change, reindexer_context)


class ElasticPillowReindexer(PillowReindexer):

    def __init__(self, pillow, change_provider, elasticsearch, index_info):
        super(ElasticPillowReindexer, self).__init__(pillow, change_provider)
        self.es = elasticsearch
        self.index_info = index_info

    def clean_index(self):
        if self.es.indices.exists(self.index_info.index):
            self.es.indices.delete(index=self.index_info.index)

    def reindex(self, start_from=None):
        if not start_from:
            # when not resuming force delete and create the index
            self._prepare_index_for_reindex()

        super(ElasticPillowReindexer, self).reindex(start_from)

        self._prepare_index_for_usage()

    def _prepare_index_for_reindex(self):
        if not self.es.indices.exists(self.index_info.index):
            self.es.indices.create(index=self.index_info.index, body=self.index_info.meta)
        initialize_mapping_if_necessary(self.es, self.index_info)
        set_index_reindex_settings(self.es, self.index_info.index)

    def _prepare_index_for_usage(self):
        set_index_normal_settings(self.es, self.index_info.index)
        self.es.indices.refresh(self.index_info.index)


def get_default_reindexer_for_elastic_pillow(pillow, change_provider):
    return ElasticPillowReindexer(
        pillow=pillow,
        change_provider=change_provider,
        elasticsearch=pillow.get_es_new(),
        index_info=get_index_info_from_pillow(pillow),
    )

{'fraction_non_alphanumeric': 0.07510460251046025, 'fraction_numerical': 0.0, 'mean_word_length': 3.6193236714975847, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6267032', 'n_tokens_mistral': 1503, 'n_tokens_neox': 1465, 'n_words': 260}