<commit_before>import copy
from urllib import unquote
import rawes
from django.conf import settings
from corehq.pillows.mappings.app_mapping import APP_INDEX
from corehq.pillows.mappings.case_mapping import CASE_INDEX
from corehq.pillows.mappings.domain_mapping import DOMAIN_INDEX
from corehq.pillows.mappings.group_mapping import GROUP_INDEX
from corehq.pillows.mappings.reportcase_mapping import REPORT_CASE_INDEX
from corehq.pillows.mappings.sms_mapping import SMS_INDEX
from corehq.pillows.mappings.tc_sms_mapping import TCSMS_INDEX
from corehq.pillows.mappings.user_mapping import USER_INDEX
from corehq.pillows.mappings.xform_mapping import XFORM_INDEX


def get_es(timeout=30):
    """
    Get a handle to the configured elastic search DB
    """
    return rawes.Elastic('%s:%s' % (settings.ELASTICSEARCH_HOST, 
                                    settings.ELASTICSEARCH_PORT),
                         timeout=timeout)


ES_URLS = {
    "forms": XFORM_INDEX + '/xform/_search',
    "cases": CASE_INDEX + '/case/_search',
    "active_cases": CASE_INDEX + '/case/_search',
    "users": USER_INDEX + '/user/_search',
    "users_all": USER_INDEX + '/user/_search',
    "domains": DOMAIN_INDEX + '/hqdomain/_search',
    "apps": APP_INDEX + '/app/_search',
    "groups": GROUP_INDEX + '/group/_search',
    "sms": SMS_INDEX + '/sms/_search',
    "tc_sms": TCSMS_INDEX + '/tc_sms/_search',
    "report_cases": REPORT_CASE_INDEX + '/report_case/_search',
}

ADD_TO_ES_FILTER = {
    "forms": [
        {"term": {"doc_type": "xforminstance"}},
        {"not": {"missing": {"field": "xmlns"}}},
        {"not": {"missing": {"field": "form.meta.userID"}}},
    ],
    "users": [
        {"term": {"doc_type": "CommCareUser"}},
        {"term": {"base_doc": "couchuser"}},
        {"term": {"is_active": True}},
    ],
    "active_cases": [
        {"term": {"closed": False}},
    ],
}<commit_msg>apply default user filters in new indicators<commit_after>import copy
from urllib import unquote
import rawes
from django.conf import settings
from corehq.pillows.mappings.app_mapping import APP_INDEX
from corehq.pillows.mappings.case_mapping import CASE_INDEX
from corehq.pillows.mappings.domain_mapping import DOMAIN_INDEX
from corehq.pillows.mappings.group_mapping import GROUP_INDEX
from corehq.pillows.mappings.reportcase_mapping import REPORT_CASE_INDEX
from corehq.pillows.mappings.sms_mapping import SMS_INDEX
from corehq.pillows.mappings.tc_sms_mapping import TCSMS_INDEX
from corehq.pillows.mappings.user_mapping import USER_INDEX
from corehq.pillows.mappings.xform_mapping import XFORM_INDEX


def get_es(timeout=30):
    """
    Get a handle to the configured elastic search DB
    """
    return rawes.Elastic('%s:%s' % (settings.ELASTICSEARCH_HOST, 
                                    settings.ELASTICSEARCH_PORT),
                         timeout=timeout)


ES_URLS = {
    "forms": XFORM_INDEX + '/xform/_search',
    "cases": CASE_INDEX + '/case/_search',
    "active_cases": CASE_INDEX + '/case/_search',
    "users": USER_INDEX + '/user/_search',
    "users_all": USER_INDEX + '/user/_search',
    "domains": DOMAIN_INDEX + '/hqdomain/_search',
    "apps": APP_INDEX + '/app/_search',
    "groups": GROUP_INDEX + '/group/_search',
    "sms": SMS_INDEX + '/sms/_search',
    "tc_sms": TCSMS_INDEX + '/tc_sms/_search',
    "report_cases": REPORT_CASE_INDEX + '/report_case/_search',
}

ADD_TO_ES_FILTER = {
    "forms": [
        {"term": {"doc_type": "xforminstance"}},
        {"not": {"missing": {"field": "xmlns"}}},
        {"not": {"missing": {"field": "form.meta.userID"}}},
    ],
    "users": [
        {"term": {"doc_type": "CommCareUser"}},
        {"term": {"base_doc": "couchuser"}},
        {"term": {"is_active": True}},
    ],
    "users_all": [
        {"term": {"base_doc": "couchuser"}},
        {"term": {"is_active": True}},
    ],
    "active_cases": [
        {"term": {"closed": False}},
    ],
}
{'fraction_non_alphanumeric': 0.149428208386277, 'fraction_numerical': 0.0010165184243964421, 'mean_word_length': 3.5555555555555554, 'pattern_counts': {'":': 65, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6390865', 'n_tokens_mistral': 1363, 'n_tokens_neox': 1316, 'n_words': 263}