<commit_before>import copy
import logging
from collections import namedtuple
from urllib import unquote
from elasticsearch import Elasticsearch
from django.conf import settings
from elasticsearch.exceptions import ElasticsearchException

from corehq.apps.es.utils import flatten_field_dict
from corehq.pillows.mappings.ledger_mapping import LEDGER_INDEX_INFO
from corehq.pillows.mappings.reportxform_mapping import REPORT_XFORM_INDEX
from pillowtop.processors.elastic import send_to_elasticsearch as send_to_es
from corehq.pillows.mappings.app_mapping import APP_INDEX
from corehq.pillows.mappings.case_mapping import CASE_INDEX
from corehq.pillows.mappings.case_search_mapping import CASE_SEARCH_INDEX_INFO
from corehq.pillows.mappings.domain_mapping import DOMAIN_INDEX_INFO
from corehq.pillows.mappings.group_mapping import GROUP_INDEX_INFO
from corehq.pillows.mappings.reportcase_mapping import REPORT_CASE_INDEX
from corehq.pillows.mappings.sms_mapping import SMS_INDEX_INFO
from corehq.pillows.mappings.user_mapping import USER_INDEX_INFO
from corehq.pillows.mappings.xform_mapping import XFORM_INDEX_INFO


def get_es_new(**kwargs):
    """
    Get a handle to the configured elastic search DB.
    Returns an elasticsearch.Elasticsearch instance.
    """
    es_hosts = getattr(settings, 'ELASTICSEARCH_HOSTS', None)
    if not es_hosts:
        es_hosts = [settings.ELASTICSEARCH_HOST]

    hosts = [
        {
            'host': host,
            'port': settings.ELASTICSEARCH_PORT,
        }
        for host in es_hosts
    ]
    return Elasticsearch(hosts, **kwargs)


def doc_exists_in_es(index_info, doc_id_or_dict):
    """
    Check if a document exists, by ID or the whole document.
    """
    if isinstance(doc_id_or_dict, basestring):
        doc_id = doc_id_or_dict
    else:<commit_msg>don't re-create the ES client on every call<commit_after>import copy
import logging
from collections import namedtuple
from urllib import unquote
from elasticsearch import Elasticsearch
from django.conf import settings
from elasticsearch.exceptions import ElasticsearchException

from corehq.apps.es.utils import flatten_field_dict
from corehq.pillows.mappings.ledger_mapping import LEDGER_INDEX_INFO
from corehq.pillows.mappings.reportxform_mapping import REPORT_XFORM_INDEX
from pillowtop.processors.elastic import send_to_elasticsearch as send_to_es
from corehq.pillows.mappings.app_mapping import APP_INDEX
from corehq.pillows.mappings.case_mapping import CASE_INDEX
from corehq.pillows.mappings.case_search_mapping import CASE_SEARCH_INDEX_INFO
from corehq.pillows.mappings.domain_mapping import DOMAIN_INDEX_INFO
from corehq.pillows.mappings.group_mapping import GROUP_INDEX_INFO
from corehq.pillows.mappings.reportcase_mapping import REPORT_CASE_INDEX
from corehq.pillows.mappings.sms_mapping import SMS_INDEX_INFO
from corehq.pillows.mappings.user_mapping import USER_INDEX_INFO
from corehq.pillows.mappings.xform_mapping import XFORM_INDEX_INFO


def get_es_new(**kwargs):
    """
    Get a handle to the configured elastic search DB.
    Returns an elasticsearch.Elasticsearch instance.
    """
    if not get_es_new._es_client:
        es_hosts = getattr(settings, 'ELASTICSEARCH_HOSTS', None)
        if not es_hosts:
            es_hosts = [settings.ELASTICSEARCH_HOST]

        hosts = [
            {
                'host': host,
                'port': settings.ELASTICSEARCH_PORT,
            }
            for host in es_hosts
        ]
        get_es_new._es_client = Elasticsearch(hosts, **kwargs)
    return get_es_new._es_client


def doc_exists_in_es(index_info, doc_id_or_dict):
    """
    Check if a document exists, by ID or the whole document.
    """
    if isinstance(doc_id_or_dict, basestring):
        doc_id = doc_id_or_dict
    else:
{'fraction_non_alphanumeric': 0.05793650793650794, 'fraction_numerical': 0.0, 'mean_word_length': 4.370738636363637, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '718568', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1193, 'n_words': 279}